From 7c5f9099c4e19149d529cf922ce0f042cfc72087 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 31 Jan 2023 19:21:43 +0000 Subject: [PATCH 01/43] record xhr --- .../rrweb/src/plugins/network/record/index.ts | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 packages/rrweb/src/plugins/network/record/index.ts diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts new file mode 100644 index 0000000000..082d2109a4 --- /dev/null +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -0,0 +1,184 @@ +/* eslint-disable no-inner-declarations */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/unbound-method */ + +import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; + +type NetworkRecordOptions = { + captureHeaders?: boolean; + captureBody?: boolean; + capturePerformance?: boolean; +}; + +const defaultNetworkOptions: NetworkRecordOptions = { + captureHeaders: false, + captureBody: false, + capturePerformance: true, +}; + +export type NetworkData = { + method: string; + url: string; + status: number; + requestHeaders?: Record; + responseHeaders?: Record; + requestBody?: any; + responseBody?: any; + performance?: PerformanceResourceTiming; +}; + +type networkCallback = (p: NetworkData) => void; + +function initNetworkObserver( + cb: networkCallback, + win: IWindow, // top window or in an iframe + options: NetworkRecordOptions, +): listenerHandler { + const networkOptions = options + ? Object.assign({}, defaultNetworkOptions, options) + : defaultNetworkOptions; + + // Check if performance is supported + if (!('performance' in win)) { + networkOptions.capturePerformance = false; + } + + let networkCount = 1; + const cancelHandlers: listenerHandler[] = []; + + // Store a reference to the native method + const originalOpen = XMLHttpRequest.prototype.open; + // Overwrite the native method + XMLHttpRequest.prototype.open = function ( + method, + url, + async = true, + username = undefined, + password = undefined, + ) { + networkCount += 1; + const markName = `xhr-${networkCount}`; + const that = this as XMLHttpRequest; + + const requestUrl: string = typeof url === 'string' ? url : url.toString(); + const requestHeaders: Record = {}; + const responseHeaders: Record = {}; + let requestBody: any = null; + let responseBody: any = null; + let status: number | undefined; + let performance: PerformanceResourceTiming | undefined; + + // Capture set request headers + if (networkOptions.captureHeaders) { + const originalSetRequestHeader = that.setRequestHeader; + that.setRequestHeader = function (header: string, value: string) { + requestHeaders[header] = value; + originalSetRequestHeader.call(this, header, value); + }; + } + + // Capture sent request body + const originalSend = that.send; + that.send = function (body) { + if (networkOptions.captureBody && typeof body !== 'undefined') { + requestBody = body; + } + // Set a mark before we trigger the XHR so we can find the performance data easier + win.performance.mark(markName); + originalSend.call(this, body); + }; + + // Capture recieved response body + if (networkOptions.captureBody) { + function onLoad() { + responseBody = that.response; + } + cancelHandlers.push(() => { + that.removeEventListener('load', onLoad); + }); + } + + function onLoadend() { + // Get the raw header string + const rawHeaders = that.getAllResponseHeaders(); + + // Convert the header string into an array of individual headers + const headers = rawHeaders.trim().split(/[\r\n]+/); + + // Create a map of header names to values + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (typeof header === 'string') { + responseHeaders[header] = value; + } + }); + + // Set response status + status = that.status; + + // Get performance data + const marks = win.performance.getEntries() as PerformanceResourceTiming[]; + const markIndex = marks.findIndex((mark) => mark.name === markName); + if (markIndex >= 0) { + performance = marks.find((mark, index) => { + return ( + index >= markIndex && + mark.initiatorType === 'xmlhttprequest' && + mark.name?.includes(requestUrl) + ); + }); + } + + // Clear performance mark + win.performance.clearMarks(markName); + + const networkData: NetworkData = { + method, + url: requestUrl, + status, + ...(networkOptions.captureHeaders && { + requestHeaders, + responseHeaders, + }), + ...(networkOptions.captureBody && { + requestBody, + responseBody, + }), + performance, + }; + + cb(networkData); + } + + that.addEventListener('loadend', onLoadend); + cancelHandlers.push(() => { + that.removeEventListener('loadend', onLoadend); + }); + + // Call the stored reference to the native method + originalOpen.call( + that, + method, + url, + async as boolean, + username as string | null | undefined, + password as string | null | undefined, + ); + }; + + return () => { + cancelHandlers.forEach((h) => h()); + }; +} + +export const PLUGIN_NAME = 'rrweb/network@2'; + +export const getRecordConsolePlugin: ( + options?: NetworkRecordOptions, +) => RecordPlugin = (options) => ({ + name: PLUGIN_NAME, + observer: initNetworkObserver, + options: options, +}); From ada77afd132d439707686280c1885fc4dfba0f12 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 13:36:40 +0000 Subject: [PATCH 02/43] use performance --- packages/rrweb/src/entries/all.ts | 2 + .../rrweb/src/plugins/network/record/index.ts | 242 ++++++------------ .../rrweb/src/plugins/network/replay/index.ts | 63 +++++ 3 files changed, 150 insertions(+), 157 deletions(-) create mode 100644 packages/rrweb/src/plugins/network/replay/index.ts diff --git a/packages/rrweb/src/entries/all.ts b/packages/rrweb/src/entries/all.ts index d67ff92447..ccfa14f69f 100644 --- a/packages/rrweb/src/entries/all.ts +++ b/packages/rrweb/src/entries/all.ts @@ -2,3 +2,5 @@ export * from '../index'; export * from '../packer'; export * from '../plugins/console/record'; export * from '../plugins/console/replay'; +export * from '../plugins/network/record'; +export * from '../plugins/network/replay'; diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 082d2109a4..d294bbe03b 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,184 +1,112 @@ -/* eslint-disable no-inner-declarations */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/unbound-method */ - import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; +export type InitiatorType = + | 'audio' + | 'beacon' + | 'body' + | 'css' + | 'early-hint' + | 'embed' + | 'fetch' + | 'frame' + | 'iframe' + | 'icon' + | 'image' + | 'img' + | 'input' + | 'link' + | 'navigation' + | 'object' + | 'ping' + | 'script' + | 'track' + | 'video' + | 'xmlhttprequest'; + type NetworkRecordOptions = { - captureHeaders?: boolean; - captureBody?: boolean; - capturePerformance?: boolean; + initiatorType?: InitiatorType[]; + captureInitialEvents?: boolean; }; const defaultNetworkOptions: NetworkRecordOptions = { - captureHeaders: false, - captureBody: false, - capturePerformance: true, -}; - -export type NetworkData = { - method: string; - url: string; - status: number; - requestHeaders?: Record; - responseHeaders?: Record; - requestBody?: any; - responseBody?: any; - performance?: PerformanceResourceTiming; + initiatorType: [ + 'audio', + 'beacon', + 'body', + 'css', + 'early-hint', + 'embed', + 'fetch', + 'frame', + 'iframe', + 'icon', + 'image', + 'img', + 'input', + 'link', + 'navigation', + 'object', + 'ping', + 'script', + 'track', + 'video', + 'xmlhttprequest', + ], + captureInitialEvents: false, }; -type networkCallback = (p: NetworkData) => void; +type networkCallback = (p: PerformanceResourceTiming) => void; function initNetworkObserver( cb: networkCallback, win: IWindow, // top window or in an iframe options: NetworkRecordOptions, ): listenerHandler { - const networkOptions = options + const networkOptions = (options ? Object.assign({}, defaultNetworkOptions, options) - : defaultNetworkOptions; - - // Check if performance is supported + : defaultNetworkOptions) as { + initiatorType: InitiatorType[]; + captureInitialEvents: boolean; + }; if (!('performance' in win)) { - networkOptions.capturePerformance = false; - } - - let networkCount = 1; - const cancelHandlers: listenerHandler[] = []; - - // Store a reference to the native method - const originalOpen = XMLHttpRequest.prototype.open; - // Overwrite the native method - XMLHttpRequest.prototype.open = function ( - method, - url, - async = true, - username = undefined, - password = undefined, - ) { - networkCount += 1; - const markName = `xhr-${networkCount}`; - const that = this as XMLHttpRequest; - - const requestUrl: string = typeof url === 'string' ? url : url.toString(); - const requestHeaders: Record = {}; - const responseHeaders: Record = {}; - let requestBody: any = null; - let responseBody: any = null; - let status: number | undefined; - let performance: PerformanceResourceTiming | undefined; - - // Capture set request headers - if (networkOptions.captureHeaders) { - const originalSetRequestHeader = that.setRequestHeader; - that.setRequestHeader = function (header: string, value: string) { - requestHeaders[header] = value; - originalSetRequestHeader.call(this, header, value); - }; - } - - // Capture sent request body - const originalSend = that.send; - that.send = function (body) { - if (networkOptions.captureBody && typeof body !== 'undefined') { - requestBody = body; - } - // Set a mark before we trigger the XHR so we can find the performance data easier - win.performance.mark(markName); - originalSend.call(this, body); + return () => { + // }; - - // Capture recieved response body - if (networkOptions.captureBody) { - function onLoad() { - responseBody = that.response; - } - cancelHandlers.push(() => { - that.removeEventListener('load', onLoad); - }); - } - - function onLoadend() { - // Get the raw header string - const rawHeaders = that.getAllResponseHeaders(); - - // Convert the header string into an array of individual headers - const headers = rawHeaders.trim().split(/[\r\n]+/); - - // Create a map of header names to values - headers.forEach((line) => { - const parts = line.split(': '); - const header = parts.shift(); - const value = parts.join(': '); - if (typeof header === 'string') { - responseHeaders[header] = value; - } - }); - - // Set response status - status = that.status; - - // Get performance data - const marks = win.performance.getEntries() as PerformanceResourceTiming[]; - const markIndex = marks.findIndex((mark) => mark.name === markName); - if (markIndex >= 0) { - performance = marks.find((mark, index) => { - return ( - index >= markIndex && - mark.initiatorType === 'xmlhttprequest' && - mark.name?.includes(requestUrl) - ); - }); - } - - // Clear performance mark - win.performance.clearMarks(markName); - - const networkData: NetworkData = { - method, - url: requestUrl, - status, - ...(networkOptions.captureHeaders && { - requestHeaders, - responseHeaders, - }), - ...(networkOptions.captureBody && { - requestBody, - responseBody, - }), - performance, - }; - - cb(networkData); - } - - that.addEventListener('loadend', onLoadend); - cancelHandlers.push(() => { - that.removeEventListener('loadend', onLoadend); - }); - - // Call the stored reference to the native method - originalOpen.call( - that, - method, - url, - async as boolean, - username as string | null | undefined, - password as string | null | undefined, - ); - }; - + } + if (networkOptions.captureInitialEvents) { + const initialResources = win.performance.getEntriesByType( + 'resource', + ) as PerformanceResourceTiming[]; + initialResources + .filter((resource) => + networkOptions.initiatorType.includes( + resource.initiatorType as InitiatorType, + ), + ) + .forEach((resource) => cb(resource)); + } + const observer = new win.PerformanceObserver((entries) => { + const resources = entries.getEntries() as PerformanceResourceTiming[]; + resources + .filter((resource) => + networkOptions.initiatorType.includes( + resource.initiatorType as InitiatorType, + ), + ) + .forEach((resource) => cb(resource)); + }); + observer.observe({ type: 'resource' }); return () => { - cancelHandlers.forEach((h) => h()); + observer.disconnect(); }; } -export const PLUGIN_NAME = 'rrweb/network@2'; +export const NETWORK_PLUGIN_NAME = 'rrweb/network@1'; -export const getRecordConsolePlugin: ( +export const getRecordNetworkPlugin: ( options?: NetworkRecordOptions, ) => RecordPlugin = (options) => ({ - name: PLUGIN_NAME, + name: NETWORK_PLUGIN_NAME, observer: initNetworkObserver, options: options, }); diff --git a/packages/rrweb/src/plugins/network/replay/index.ts b/packages/rrweb/src/plugins/network/replay/index.ts new file mode 100644 index 0000000000..ad9ba7d22c --- /dev/null +++ b/packages/rrweb/src/plugins/network/replay/index.ts @@ -0,0 +1,63 @@ +import { InitiatorType, NETWORK_PLUGIN_NAME } from '../record'; +import type { eventWithTime } from '@rrweb/types'; +import { EventType } from '@rrweb/types'; +import type { ReplayPlugin } from '../../../types'; + +export type NetworkReplayLogger = (data: PerformanceResourceTiming) => void; + +export type NetworkReplayOptions = { + initiatorType?: InitiatorType[]; + replayLogger: NetworkReplayLogger; +}; + +const defaultNetworkOptions = { + initiatorType: [ + 'audio', + 'beacon', + 'body', + 'css', + 'early-hint', + 'embed', + 'fetch', + 'frame', + 'iframe', + 'icon', + 'image', + 'img', + 'input', + 'link', + 'navigation', + 'object', + 'ping', + 'script', + 'track', + 'video', + 'xmlhttprequest', + ], +}; + +export const getReplayNetworkPlugin: ( + options: NetworkReplayOptions, +) => ReplayPlugin = (options) => { + const networkOptions = Object.assign({}, defaultNetworkOptions, options) as { + initiatorType: InitiatorType[]; + replayLogger: NetworkReplayLogger; + }; + return { + handler(event: eventWithTime) { + if ( + event.type === EventType.Plugin && + event.data.plugin === NETWORK_PLUGIN_NAME + ) { + const networkData = event.data.payload as PerformanceResourceTiming; + if ( + networkOptions.initiatorType.includes( + networkData.initiatorType as InitiatorType, + ) + ) { + networkOptions.replayLogger(networkData); + } + } + }, + }; +}; From 2b23db5a8fefbe06443d422fe4a687ba64a8a59a Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 15:25:31 +0000 Subject: [PATCH 03/43] emit less events & isInitial flag --- .../rrweb/src/plugins/network/record/index.ts | 75 ++++++++++++------- .../rrweb/src/plugins/network/replay/index.ts | 20 ++--- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index d294bbe03b..b2ea472d69 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -55,45 +55,45 @@ const defaultNetworkOptions: NetworkRecordOptions = { captureInitialEvents: false, }; -type networkCallback = (p: PerformanceResourceTiming) => void; +export type NetworkData = { + resourceTimings: PerformanceResourceTiming[]; + isInitial?: boolean; +}; -function initNetworkObserver( +type networkCallback = (data: NetworkData) => void; + +function initPerformanceObserver( cb: networkCallback, - win: IWindow, // top window or in an iframe - options: NetworkRecordOptions, -): listenerHandler { - const networkOptions = (options - ? Object.assign({}, defaultNetworkOptions, options) - : defaultNetworkOptions) as { + win: IWindow, + options: { initiatorType: InitiatorType[]; captureInitialEvents: boolean; - }; + }, +) { if (!('performance' in win)) { return () => { // }; } - if (networkOptions.captureInitialEvents) { - const initialResources = win.performance.getEntriesByType( - 'resource', - ) as PerformanceResourceTiming[]; - initialResources - .filter((resource) => - networkOptions.initiatorType.includes( - resource.initiatorType as InitiatorType, - ), - ) - .forEach((resource) => cb(resource)); + const isResourceTiming = ( + entry: PerformanceEntry, + ): entry is PerformanceResourceTiming => { + return entry.entryType === 'resource'; + }; + const getResourceTimings = (entries: PerformanceEntryList) => { + return entries.filter((entry): entry is PerformanceResourceTiming => { + return isResourceTiming(entry); + }); + }; + if (options.captureInitialEvents) { + const initialResourceTimings = getResourceTimings( + win.performance.getEntriesByType('resource'), + ); + cb({ resourceTimings: initialResourceTimings, isInitial: true }); } const observer = new win.PerformanceObserver((entries) => { - const resources = entries.getEntries() as PerformanceResourceTiming[]; - resources - .filter((resource) => - networkOptions.initiatorType.includes( - resource.initiatorType as InitiatorType, - ), - ) - .forEach((resource) => cb(resource)); + const resourceTimings = getResourceTimings(entries.getEntries()); + cb({ resourceTimings }); }); observer.observe({ type: 'resource' }); return () => { @@ -101,6 +101,25 @@ function initNetworkObserver( }; } +function initNetworkObserver( + cb: networkCallback, + win: IWindow, // top window or in an iframe + options: NetworkRecordOptions, +): listenerHandler { + const networkOptions = (options + ? Object.assign({}, defaultNetworkOptions, options) + : defaultNetworkOptions) as { + initiatorType: InitiatorType[]; + captureInitialEvents: boolean; + }; + + const performanceObserver = initPerformanceObserver(cb, win, networkOptions); + + return () => { + performanceObserver(); + }; +} + export const NETWORK_PLUGIN_NAME = 'rrweb/network@1'; export const getRecordNetworkPlugin: ( diff --git a/packages/rrweb/src/plugins/network/replay/index.ts b/packages/rrweb/src/plugins/network/replay/index.ts index ad9ba7d22c..428771c433 100644 --- a/packages/rrweb/src/plugins/network/replay/index.ts +++ b/packages/rrweb/src/plugins/network/replay/index.ts @@ -1,9 +1,9 @@ -import { InitiatorType, NETWORK_PLUGIN_NAME } from '../record'; +import { InitiatorType, NetworkData, NETWORK_PLUGIN_NAME } from '../record'; import type { eventWithTime } from '@rrweb/types'; import { EventType } from '@rrweb/types'; import type { ReplayPlugin } from '../../../types'; -export type NetworkReplayLogger = (data: PerformanceResourceTiming) => void; +export type NetworkReplayLogger = (data: NetworkData) => void; export type NetworkReplayOptions = { initiatorType?: InitiatorType[]; @@ -49,14 +49,14 @@ export const getReplayNetworkPlugin: ( event.type === EventType.Plugin && event.data.plugin === NETWORK_PLUGIN_NAME ) { - const networkData = event.data.payload as PerformanceResourceTiming; - if ( - networkOptions.initiatorType.includes( - networkData.initiatorType as InitiatorType, - ) - ) { - networkOptions.replayLogger(networkData); - } + const networkData = event.data.payload as NetworkData; + networkData.resourceTimings = networkData.resourceTimings.filter( + (resourceTiming) => + networkOptions.initiatorType.includes( + resourceTiming.initiatorType as InitiatorType, + ), + ); + networkOptions.replayLogger(networkData); } }, }; From dafbdd4b4d040de4963cb2d3b6993e7818c84392 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 15:29:38 +0000 Subject: [PATCH 04/43] stub xhr & fetch observers --- .../rrweb/src/plugins/network/record/index.ts | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index b2ea472d69..57baed7d78 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -82,7 +82,11 @@ function initPerformanceObserver( }; const getResourceTimings = (entries: PerformanceEntryList) => { return entries.filter((entry): entry is PerformanceResourceTiming => { - return isResourceTiming(entry); + return ( + isResourceTiming(entry) && + entry.initiatorType !== 'xmlhttprequest' && + entry.initiatorType !== 'fetch' + ); }); }; if (options.captureInitialEvents) { @@ -101,6 +105,26 @@ function initPerformanceObserver( }; } +function initXhrObserver( + cb: networkCallback, + win: IWindow, + options: NetworkRecordOptions, +): listenerHandler { + return () => { + // TODO: + }; +} + +function initFetchObserver( + cb: networkCallback, + win: IWindow, + options: NetworkRecordOptions, +): listenerHandler { + return () => { + // TODO: + }; +} + function initNetworkObserver( cb: networkCallback, win: IWindow, // top window or in an iframe @@ -114,9 +138,19 @@ function initNetworkObserver( }; const performanceObserver = initPerformanceObserver(cb, win, networkOptions); + let xhrObserver: listenerHandler | undefined; + if (networkOptions.initiatorType.includes('xmlhttprequest')) { + xhrObserver = initXhrObserver(cb, win, networkOptions); + } + let fetchObserver: listenerHandler | undefined; + if (networkOptions.initiatorType.includes('fetch')) { + fetchObserver = initFetchObserver(cb, win, networkOptions); + } return () => { performanceObserver(); + xhrObserver?.(); + fetchObserver?.(); }; } From 6e9377ae0601f102ef292bf175f652c4c4e62915 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 15:54:08 +0000 Subject: [PATCH 05/43] stub req headers & body --- .../rrweb/src/plugins/console/record/index.ts | 19 +------ .../rrweb/src/plugins/network/record/index.ts | 49 +++++++++++++++++-- .../rrweb/src/plugins/network/replay/index.ts | 9 ++-- .../record => utils}/error-stack-parser.ts | 0 .../{console/record => utils}/stringify.ts | 16 +++++- 5 files changed, 66 insertions(+), 27 deletions(-) rename packages/rrweb/src/plugins/{console/record => utils}/error-stack-parser.ts (100%) rename packages/rrweb/src/plugins/{console/record => utils}/stringify.ts (91%) diff --git a/packages/rrweb/src/plugins/console/record/index.ts b/packages/rrweb/src/plugins/console/record/index.ts index 47cd513695..9648b24d3d 100644 --- a/packages/rrweb/src/plugins/console/record/index.ts +++ b/packages/rrweb/src/plugins/console/record/index.ts @@ -1,22 +1,7 @@ import type { listenerHandler, RecordPlugin, IWindow } from '@rrweb/types'; import { patch } from '../../../utils'; -import { ErrorStackParser, StackFrame } from './error-stack-parser'; -import { stringify } from './stringify'; - -export type StringifyOptions = { - // limit of string length - stringLengthLimit?: number; - /** - * limit of number of keys in an object - * if an object contains more keys than this limit, we would call its toString function directly - */ - numOfKeysLimit: number; - /** - * limit number of depth in an object - * if an object is too deep, toString process may cause browser OOM - */ - depthOfLimit: number; -}; +import { ErrorStackParser, StackFrame } from '../../utils/error-stack-parser'; +import { stringify, StringifyOptions } from '../../utils/stringify'; type LogRecordOptions = { level?: LogLevel[]; diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 57baed7d78..975f329b19 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,4 +1,5 @@ import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; +import type { StringifyOptions } from '../../utils/stringify'; export type InitiatorType = | 'audio' @@ -25,6 +26,20 @@ export type InitiatorType = type NetworkRecordOptions = { initiatorType?: InitiatorType[]; + captureHeaders?: + | boolean + | StringifyOptions + | { + request: boolean | StringifyOptions; + response: boolean | StringifyOptions; + }; + captureBody?: + | boolean + | StringifyOptions + | { + request: boolean | StringifyOptions; + response: boolean | StringifyOptions; + }; captureInitialEvents?: boolean; }; @@ -52,11 +67,32 @@ const defaultNetworkOptions: NetworkRecordOptions = { 'video', 'xmlhttprequest', ], + captureHeaders: false, + captureBody: false, captureInitialEvents: false, }; +type Headers = Record; +type Body = + | string + | Document + | Blob + | ArrayBufferView + | ArrayBuffer + | FormData + | URLSearchParams + | ReadableStream + | null + | undefined; + export type NetworkData = { - resourceTimings: PerformanceResourceTiming[]; + requests: { + resourceTiming: PerformanceResourceTiming; + requestHeaders?: Headers; + requestBody?: Body; + responseHeaders?: Headers; + responseBody?: Body; + }[]; isInitial?: boolean; }; @@ -93,11 +129,18 @@ function initPerformanceObserver( const initialResourceTimings = getResourceTimings( win.performance.getEntriesByType('resource'), ); - cb({ resourceTimings: initialResourceTimings, isInitial: true }); + cb({ + requests: initialResourceTimings.map((resourceTiming) => ({ + resourceTiming, + })), + isInitial: true, + }); } const observer = new win.PerformanceObserver((entries) => { const resourceTimings = getResourceTimings(entries.getEntries()); - cb({ resourceTimings }); + cb({ + requests: resourceTimings.map((resourceTiming) => ({ resourceTiming })), + }); }); observer.observe({ type: 'resource' }); return () => { diff --git a/packages/rrweb/src/plugins/network/replay/index.ts b/packages/rrweb/src/plugins/network/replay/index.ts index 428771c433..8977be7899 100644 --- a/packages/rrweb/src/plugins/network/replay/index.ts +++ b/packages/rrweb/src/plugins/network/replay/index.ts @@ -50,11 +50,10 @@ export const getReplayNetworkPlugin: ( event.data.plugin === NETWORK_PLUGIN_NAME ) { const networkData = event.data.payload as NetworkData; - networkData.resourceTimings = networkData.resourceTimings.filter( - (resourceTiming) => - networkOptions.initiatorType.includes( - resourceTiming.initiatorType as InitiatorType, - ), + networkData.requests = networkData.requests.filter((request) => + networkOptions.initiatorType.includes( + request.resourceTiming.initiatorType as InitiatorType, + ), ); networkOptions.replayLogger(networkData); } diff --git a/packages/rrweb/src/plugins/console/record/error-stack-parser.ts b/packages/rrweb/src/plugins/utils/error-stack-parser.ts similarity index 100% rename from packages/rrweb/src/plugins/console/record/error-stack-parser.ts rename to packages/rrweb/src/plugins/utils/error-stack-parser.ts diff --git a/packages/rrweb/src/plugins/console/record/stringify.ts b/packages/rrweb/src/plugins/utils/stringify.ts similarity index 91% rename from packages/rrweb/src/plugins/console/record/stringify.ts rename to packages/rrweb/src/plugins/utils/stringify.ts index eef8c38a40..5598de5567 100644 --- a/packages/rrweb/src/plugins/console/record/stringify.ts +++ b/packages/rrweb/src/plugins/utils/stringify.ts @@ -2,8 +2,20 @@ * this file is used to serialize log message to string * */ - -import type { StringifyOptions } from './index'; +export type StringifyOptions = { + // limit of string length + stringLengthLimit?: number; + /** + * limit of number of keys in an object + * if an object contains more keys than this limit, we would call its toString function directly + */ + numOfKeysLimit: number; + /** + * limit number of depth in an object + * if an object is too deep, toString process may cause browser OOM + */ + depthOfLimit: number; +}; /** * transfer the node path in Event to string From 27f436050b02cd499779f2df55f170f94d00a92f Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 15:58:31 +0000 Subject: [PATCH 06/43] small type refactor --- .../rrweb/src/plugins/network/record/index.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 975f329b19..3586914edb 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -85,14 +85,16 @@ type Body = | null | undefined; +type NetworkRequest = { + resourceTiming: PerformanceResourceTiming; + requestHeaders?: Headers; + requestBody?: Body; + responseHeaders?: Headers; + responseBody?: Body; +}; + export type NetworkData = { - requests: { - resourceTiming: PerformanceResourceTiming; - requestHeaders?: Headers; - requestBody?: Body; - responseHeaders?: Headers; - responseBody?: Body; - }[]; + requests: NetworkRequest[]; isInitial?: boolean; }; From 7fa75b50256a6f271fd8970ec4b1da7eb3c16e48 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 16:38:30 +0000 Subject: [PATCH 07/43] add NavigationTiming --- .../rrweb/src/plugins/network/record/index.ts | 36 +++++++++------- .../rrweb/src/plugins/network/replay/index.ts | 42 ++----------------- 2 files changed, 23 insertions(+), 55 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 3586914edb..24fa87cab9 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -86,7 +86,7 @@ type Body = | undefined; type NetworkRequest = { - resourceTiming: PerformanceResourceTiming; + performanceEntry: PerformanceEntry; requestHeaders?: Headers; requestBody?: Body; responseHeaders?: Headers; @@ -113,38 +113,42 @@ function initPerformanceObserver( // }; } + const isNavigationTiming = ( + entry: PerformanceEntry, + ): entry is PerformanceNavigationTiming => entry.entryType === 'navigation'; const isResourceTiming = ( entry: PerformanceEntry, - ): entry is PerformanceResourceTiming => { - return entry.entryType === 'resource'; - }; - const getResourceTimings = (entries: PerformanceEntryList) => { - return entries.filter((entry): entry is PerformanceResourceTiming => { + ): entry is PerformanceResourceTiming => entry.entryType === 'resource'; + const getPerformanceEntries = (entries: PerformanceEntryList) => { + return entries.filter((entry) => { return ( - isResourceTiming(entry) && - entry.initiatorType !== 'xmlhttprequest' && - entry.initiatorType !== 'fetch' + isNavigationTiming(entry) || + (isResourceTiming(entry) && + entry.initiatorType !== 'xmlhttprequest' && + entry.initiatorType !== 'fetch') ); }); }; if (options.captureInitialEvents) { - const initialResourceTimings = getResourceTimings( - win.performance.getEntriesByType('resource'), + const initialPerformanceEntries = getPerformanceEntries( + win.performance.getEntries(), ); cb({ - requests: initialResourceTimings.map((resourceTiming) => ({ - resourceTiming, + requests: initialPerformanceEntries.map((performanceEntry) => ({ + performanceEntry, })), isInitial: true, }); } const observer = new win.PerformanceObserver((entries) => { - const resourceTimings = getResourceTimings(entries.getEntries()); + const performanceEntries = getPerformanceEntries(entries.getEntries()); cb({ - requests: resourceTimings.map((resourceTiming) => ({ resourceTiming })), + requests: performanceEntries.map((performanceEntry) => ({ + performanceEntry, + })), }); }); - observer.observe({ type: 'resource' }); + observer.observe({ entryTypes: ['navigation', 'resource'] }); return () => { observer.disconnect(); }; diff --git a/packages/rrweb/src/plugins/network/replay/index.ts b/packages/rrweb/src/plugins/network/replay/index.ts index 8977be7899..fde784fb8e 100644 --- a/packages/rrweb/src/plugins/network/replay/index.ts +++ b/packages/rrweb/src/plugins/network/replay/index.ts @@ -3,46 +3,15 @@ import type { eventWithTime } from '@rrweb/types'; import { EventType } from '@rrweb/types'; import type { ReplayPlugin } from '../../../types'; -export type NetworkReplayLogger = (data: NetworkData) => void; +export type NetworkReplayer = (data: NetworkData) => void; export type NetworkReplayOptions = { - initiatorType?: InitiatorType[]; - replayLogger: NetworkReplayLogger; -}; - -const defaultNetworkOptions = { - initiatorType: [ - 'audio', - 'beacon', - 'body', - 'css', - 'early-hint', - 'embed', - 'fetch', - 'frame', - 'iframe', - 'icon', - 'image', - 'img', - 'input', - 'link', - 'navigation', - 'object', - 'ping', - 'script', - 'track', - 'video', - 'xmlhttprequest', - ], + replayer: NetworkReplayer; }; export const getReplayNetworkPlugin: ( options: NetworkReplayOptions, ) => ReplayPlugin = (options) => { - const networkOptions = Object.assign({}, defaultNetworkOptions, options) as { - initiatorType: InitiatorType[]; - replayLogger: NetworkReplayLogger; - }; return { handler(event: eventWithTime) { if ( @@ -50,12 +19,7 @@ export const getReplayNetworkPlugin: ( event.data.plugin === NETWORK_PLUGIN_NAME ) { const networkData = event.data.payload as NetworkData; - networkData.requests = networkData.requests.filter((request) => - networkOptions.initiatorType.includes( - request.resourceTiming.initiatorType as InitiatorType, - ), - ); - networkOptions.replayLogger(networkData); + options.replayer(networkData); } }, }; From 52c58b349b50542535a5da0777ccda769995bbea Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 16:40:50 +0000 Subject: [PATCH 08/43] rename props --- .../rrweb/src/plugins/network/record/index.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 24fa87cab9..cb080df783 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -26,21 +26,21 @@ export type InitiatorType = type NetworkRecordOptions = { initiatorType?: InitiatorType[]; - captureHeaders?: + recordHeaders?: | boolean | StringifyOptions | { request: boolean | StringifyOptions; response: boolean | StringifyOptions; }; - captureBody?: + recordBody?: | boolean | StringifyOptions | { request: boolean | StringifyOptions; response: boolean | StringifyOptions; }; - captureInitialEvents?: boolean; + recordInitialEvents?: boolean; }; const defaultNetworkOptions: NetworkRecordOptions = { @@ -67,9 +67,9 @@ const defaultNetworkOptions: NetworkRecordOptions = { 'video', 'xmlhttprequest', ], - captureHeaders: false, - captureBody: false, - captureInitialEvents: false, + recordHeaders: false, + recordBody: false, + recordInitialEvents: false, }; type Headers = Record; @@ -105,7 +105,7 @@ function initPerformanceObserver( win: IWindow, options: { initiatorType: InitiatorType[]; - captureInitialEvents: boolean; + recordInitialEvents: boolean; }, ) { if (!('performance' in win)) { @@ -129,7 +129,7 @@ function initPerformanceObserver( ); }); }; - if (options.captureInitialEvents) { + if (options.recordInitialEvents) { const initialPerformanceEntries = getPerformanceEntries( win.performance.getEntries(), ); @@ -183,7 +183,7 @@ function initNetworkObserver( ? Object.assign({}, defaultNetworkOptions, options) : defaultNetworkOptions) as { initiatorType: InitiatorType[]; - captureInitialEvents: boolean; + recordInitialEvents: boolean; }; const performanceObserver = initPerformanceObserver(cb, win, networkOptions); From 05a45dfa8919a08a3c2728bd485f5bae986c2d9d Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 17:47:47 +0000 Subject: [PATCH 09/43] implement fetch --- .../rrweb/src/plugins/network/record/index.ts | 129 ++++++++++++++---- packages/rrweb/src/utils.ts | 12 ++ 2 files changed, 111 insertions(+), 30 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index cb080df783..46b5b1986a 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,4 +1,6 @@ +/* eslint-disable no-useless-catch */ import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; +import { findLast } from '../../../utils'; import type { StringifyOptions } from '../../utils/stringify'; export type InitiatorType = @@ -73,24 +75,13 @@ const defaultNetworkOptions: NetworkRecordOptions = { }; type Headers = Record; -type Body = - | string - | Document - | Blob - | ArrayBufferView - | ArrayBuffer - | FormData - | URLSearchParams - | ReadableStream - | null - | undefined; type NetworkRequest = { performanceEntry: PerformanceEntry; requestHeaders?: Headers; - requestBody?: Body; + requestBody?: string | null; responseHeaders?: Headers; - responseBody?: Body; + responseBody?: string | null; }; export type NetworkData = { @@ -100,13 +91,14 @@ export type NetworkData = { type networkCallback = (data: NetworkData) => void; +type NetworkObserverOptions = NetworkRecordOptions & { + initiatorType: InitiatorType[]; +}; + function initPerformanceObserver( cb: networkCallback, win: IWindow, - options: { - initiatorType: InitiatorType[]; - recordInitialEvents: boolean; - }, + options: NetworkObserverOptions, ) { if (!('performance' in win)) { return () => { @@ -157,8 +149,13 @@ function initPerformanceObserver( function initXhrObserver( cb: networkCallback, win: IWindow, - options: NetworkRecordOptions, + options: NetworkObserverOptions, ): listenerHandler { + if (!options.initiatorType.includes('xmlhttprequest')) { + return () => { + // + }; + } return () => { // TODO: }; @@ -167,10 +164,88 @@ function initXhrObserver( function initFetchObserver( cb: networkCallback, win: IWindow, - options: NetworkRecordOptions, + options: NetworkObserverOptions, ): listenerHandler { + if (!options.initiatorType.includes('fetch')) { + return () => { + // + }; + } + const originalFetch = win.fetch; + const wrappedFetch: typeof fetch = async function (url, init) { + const recordRequestHeaders = + !!options.recordHeaders && + (typeof options.recordHeaders === 'boolean' || + !('request' in options.recordHeaders) || + options.recordHeaders.request); + const recordRequestBody = + !!options.recordBody && + (typeof options.recordBody === 'boolean' || + !('request' in options.recordBody) || + options.recordBody.request); + const recordResponseHeaders = + !!options.recordHeaders && + (typeof options.recordHeaders === 'boolean' || + !('response' in options.recordHeaders) || + options.recordHeaders.response); + const recordResponseBody = + !!options.recordBody && + (typeof options.recordBody === 'boolean' || + !('response' in options.recordBody) || + options.recordBody.response); + + const req = new Request(url, init); + let performanceEntry: PerformanceResourceTiming | undefined; + const networkRequest: Partial = {}; + if (recordRequestHeaders) { + networkRequest.requestHeaders = {}; + req.headers.forEach((value, key) => { + networkRequest.requestHeaders![key] = value; + }); + } + if (recordRequestBody) { + networkRequest.requestBody = init?.body?.toString(); + if (networkRequest.requestBody === undefined) { + networkRequest.requestBody = null; + } + } + try { + const res = await originalFetch(req); + const performanceEntries = win.performance.getEntriesByType( + 'resource', + ) as PerformanceResourceTiming[]; + performanceEntry = findLast( + performanceEntries, + (p) => p.initiatorType === 'fetch' && p.name === req.url, + ); + if (recordResponseHeaders) { + networkRequest.responseHeaders = {}; + res.headers.forEach((value, key) => { + networkRequest.responseHeaders![key] = value; + }); + } + if (recordResponseBody) { + networkRequest.responseBody = await res.clone().text(); + } + return res; + } catch (cause) { + throw cause; + } finally { + if (performanceEntry) { + cb({ requests: [{ performanceEntry, ...networkRequest }] }); + } + } + }; + wrappedFetch.prototype = {}; + Object.defineProperties(wrappedFetch, { + __rrweb_original__: { + enumerable: false, + value: originalFetch, + }, + }); + win.fetch = wrappedFetch; return () => { - // TODO: + win.fetch = originalFetch; }; } @@ -187,19 +262,13 @@ function initNetworkObserver( }; const performanceObserver = initPerformanceObserver(cb, win, networkOptions); - let xhrObserver: listenerHandler | undefined; - if (networkOptions.initiatorType.includes('xmlhttprequest')) { - xhrObserver = initXhrObserver(cb, win, networkOptions); - } - let fetchObserver: listenerHandler | undefined; - if (networkOptions.initiatorType.includes('fetch')) { - fetchObserver = initFetchObserver(cb, win, networkOptions); - } + const xhrObserver = initXhrObserver(cb, win, networkOptions); + const fetchObserver = initFetchObserver(cb, win, networkOptions); return () => { performanceObserver(); - xhrObserver?.(); - fetchObserver?.(); + xhrObserver(); + fetchObserver(); }; } diff --git a/packages/rrweb/src/utils.ts b/packages/rrweb/src/utils.ts index 1d5ec83855..d61de7c588 100644 --- a/packages/rrweb/src/utils.ts +++ b/packages/rrweb/src/utils.ts @@ -545,3 +545,15 @@ export function inDom(n: Node): boolean { if (!doc) return false; return doc.contains(n) || shadowHostInDom(n); } + +export function findLast( + array: Array, + predicate: (value: T) => boolean, +): T | undefined { + const length = array.length; + for (let i = length - 1; i >= 0; i -= 1) { + if (predicate(array[i])) { + return array[i]; + } + } +} From 863717da767b05fb616684ba168575841be1d98e Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 18:00:38 +0000 Subject: [PATCH 10/43] use stringify --- .../rrweb/src/plugins/network/record/index.ts | 54 +++++++++++++------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 46b5b1986a..3d9e852163 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,7 +1,6 @@ -/* eslint-disable no-useless-catch */ import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; import { findLast } from '../../../utils'; -import type { StringifyOptions } from '../../utils/stringify'; +import { stringify, StringifyOptions } from '../../utils/stringify'; export type InitiatorType = | 'audio' @@ -194,22 +193,28 @@ function initFetchObserver( !('response' in options.recordBody) || options.recordBody.response); - const req = new Request(url, init); let performanceEntry: PerformanceResourceTiming | undefined; const networkRequest: Partial = {}; - if (recordRequestHeaders) { - networkRequest.requestHeaders = {}; - req.headers.forEach((value, key) => { - networkRequest.requestHeaders![key] = value; - }); - } - if (recordRequestBody) { - networkRequest.requestBody = init?.body?.toString(); - if (networkRequest.requestBody === undefined) { - networkRequest.requestBody = null; - } - } try { + const req = new Request(url, init); + if (recordRequestHeaders) { + networkRequest.requestHeaders = {}; + req.headers.forEach((value, key) => { + networkRequest.requestHeaders![key] = value; + }); + } + if (recordRequestBody) { + if (!req.body) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = stringify( + req.body, + typeof recordRequestBody === 'object' + ? recordRequestBody + : undefined, + ); + } + } const res = await originalFetch(req); const performanceEntries = win.performance.getEntriesByType( 'resource', @@ -225,10 +230,27 @@ function initFetchObserver( }); } if (recordResponseBody) { - networkRequest.responseBody = await res.clone().text(); + const reqBody = await res.clone().text(); + if (!reqBody) { + networkRequest.responseBody = null; + } else { + try { + const objBody = JSON.parse(reqBody) as object; + networkRequest.responseBody = stringify( + objBody, + typeof recordResponseBody === 'object' + ? recordResponseBody + : undefined, + ); + } catch { + networkRequest.responseBody = reqBody; + } + } } return res; + // eslint-disable-next-line no-useless-catch } catch (cause) { + // failed to fetch throw cause; } finally { if (performanceEntry) { From e693366aa237e809c4da3f77bff45aeb44d836be Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 18:02:36 +0000 Subject: [PATCH 11/43] refactor options --- .../rrweb/src/plugins/network/record/index.ts | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 3d9e852163..803fa75576 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -170,29 +170,29 @@ function initFetchObserver( // }; } + const recordRequestHeaders = + !!options.recordHeaders && + (typeof options.recordHeaders === 'boolean' || + !('request' in options.recordHeaders) || + options.recordHeaders.request); + const recordRequestBody = + !!options.recordBody && + (typeof options.recordBody === 'boolean' || + !('request' in options.recordBody) || + options.recordBody.request); + const recordResponseHeaders = + !!options.recordHeaders && + (typeof options.recordHeaders === 'boolean' || + !('response' in options.recordHeaders) || + options.recordHeaders.response); + const recordResponseBody = + !!options.recordBody && + (typeof options.recordBody === 'boolean' || + !('response' in options.recordBody) || + options.recordBody.response); + const originalFetch = win.fetch; const wrappedFetch: typeof fetch = async function (url, init) { - const recordRequestHeaders = - !!options.recordHeaders && - (typeof options.recordHeaders === 'boolean' || - !('request' in options.recordHeaders) || - options.recordHeaders.request); - const recordRequestBody = - !!options.recordBody && - (typeof options.recordBody === 'boolean' || - !('request' in options.recordBody) || - options.recordBody.request); - const recordResponseHeaders = - !!options.recordHeaders && - (typeof options.recordHeaders === 'boolean' || - !('response' in options.recordHeaders) || - options.recordHeaders.response); - const recordResponseBody = - !!options.recordBody && - (typeof options.recordBody === 'boolean' || - !('response' in options.recordBody) || - options.recordBody.response); - let performanceEntry: PerformanceResourceTiming | undefined; const networkRequest: Partial = {}; try { From 78b338f3d27bd813b94bf334eef6ed7047004785 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 18:41:29 +0000 Subject: [PATCH 12/43] add requestMethod --- .../rrweb/src/plugins/network/record/index.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 803fa75576..7eb6413ae8 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -77,6 +77,7 @@ type Headers = Record; type NetworkRequest = { performanceEntry: PerformanceEntry; + requestMethod: string; requestHeaders?: Headers; requestBody?: string | null; responseHeaders?: Headers; @@ -127,6 +128,7 @@ function initPerformanceObserver( cb({ requests: initialPerformanceEntries.map((performanceEntry) => ({ performanceEntry, + requestMethod: 'GET', })), isInitial: true, }); @@ -136,6 +138,7 @@ function initPerformanceObserver( cb({ requests: performanceEntries.map((performanceEntry) => ({ performanceEntry, + requestMethod: 'GET', })), }); }); @@ -195,8 +198,8 @@ function initFetchObserver( const wrappedFetch: typeof fetch = async function (url, init) { let performanceEntry: PerformanceResourceTiming | undefined; const networkRequest: Partial = {}; + const req = new Request(url, init); try { - const req = new Request(url, init); if (recordRequestHeaders) { networkRequest.requestHeaders = {}; req.headers.forEach((value, key) => { @@ -254,7 +257,15 @@ function initFetchObserver( throw cause; } finally { if (performanceEntry) { - cb({ requests: [{ performanceEntry, ...networkRequest }] }); + cb({ + requests: [ + { + performanceEntry, + requestMethod: req.method, + ...networkRequest, + }, + ], + }); } } }; From d656ac7abe6b453e98034d52b276b18748e80f9c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 18:41:52 +0000 Subject: [PATCH 13/43] rename replay prop --- packages/rrweb/src/plugins/network/replay/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rrweb/src/plugins/network/replay/index.ts b/packages/rrweb/src/plugins/network/replay/index.ts index fde784fb8e..0d639ec37f 100644 --- a/packages/rrweb/src/plugins/network/replay/index.ts +++ b/packages/rrweb/src/plugins/network/replay/index.ts @@ -1,12 +1,12 @@ -import { InitiatorType, NetworkData, NETWORK_PLUGIN_NAME } from '../record'; +import { NetworkData, NETWORK_PLUGIN_NAME } from '../record'; import type { eventWithTime } from '@rrweb/types'; import { EventType } from '@rrweb/types'; import type { ReplayPlugin } from '../../../types'; -export type NetworkReplayer = (data: NetworkData) => void; +export type OnNetworkData = (data: NetworkData) => void; export type NetworkReplayOptions = { - replayer: NetworkReplayer; + onNetworkData: OnNetworkData; }; export const getReplayNetworkPlugin: ( @@ -19,7 +19,7 @@ export const getReplayNetworkPlugin: ( event.data.plugin === NETWORK_PLUGIN_NAME ) { const networkData = event.data.payload as NetworkData; - options.replayer(networkData); + options.onNetworkData(networkData); } }, }; From 1d77b6c85eb289dec93bb58fb3d4545e004cc9b1 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 19:01:52 +0000 Subject: [PATCH 14/43] add ignoreRequestFn --- .../rrweb/src/plugins/network/record/index.ts | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 7eb6413ae8..77758bbe97 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -27,6 +27,7 @@ export type InitiatorType = type NetworkRecordOptions = { initiatorType?: InitiatorType[]; + ignoreRequestFn?: (data: NetworkRequest) => boolean; recordHeaders?: | boolean | StringifyOptions @@ -68,6 +69,7 @@ const defaultNetworkOptions: NetworkRecordOptions = { 'video', 'xmlhttprequest', ], + ignoreEventFn: () => false, recordHeaders: false, recordBody: false, recordInitialEvents: false, @@ -91,14 +93,10 @@ export type NetworkData = { type networkCallback = (data: NetworkData) => void; -type NetworkObserverOptions = NetworkRecordOptions & { - initiatorType: InitiatorType[]; -}; - function initPerformanceObserver( cb: networkCallback, win: IWindow, - options: NetworkObserverOptions, + options: Required, ) { if (!('performance' in win)) { return () => { @@ -151,7 +149,7 @@ function initPerformanceObserver( function initXhrObserver( cb: networkCallback, win: IWindow, - options: NetworkObserverOptions, + options: Required, ): listenerHandler { if (!options.initiatorType.includes('xmlhttprequest')) { return () => { @@ -166,7 +164,7 @@ function initXhrObserver( function initFetchObserver( cb: networkCallback, win: IWindow, - options: NetworkObserverOptions, + options: Required, ): listenerHandler { if (!options.initiatorType.includes('fetch')) { return () => { @@ -283,15 +281,19 @@ function initFetchObserver( } function initNetworkObserver( - cb: networkCallback, + callback: networkCallback, win: IWindow, // top window or in an iframe options: NetworkRecordOptions, ): listenerHandler { const networkOptions = (options ? Object.assign({}, defaultNetworkOptions, options) - : defaultNetworkOptions) as { - initiatorType: InitiatorType[]; - recordInitialEvents: boolean; + : defaultNetworkOptions) as Required; + + const cb: networkCallback = (data) => { + const requests = data.requests.filter((request) => + networkOptions.ignoreRequestFn(request), + ); + callback({ ...data, requests }); }; const performanceObserver = initPerformanceObserver(cb, win, networkOptions); From f3d85e22f3dc919e0b9f5f508fa604591801a410 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 19:02:02 +0000 Subject: [PATCH 15/43] rn initiatorTypes --- packages/rrweb/src/plugins/network/record/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 77758bbe97..72c163e2b1 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -26,7 +26,7 @@ export type InitiatorType = | 'xmlhttprequest'; type NetworkRecordOptions = { - initiatorType?: InitiatorType[]; + initiatorTypes?: InitiatorType[]; ignoreRequestFn?: (data: NetworkRequest) => boolean; recordHeaders?: | boolean @@ -46,7 +46,7 @@ type NetworkRecordOptions = { }; const defaultNetworkOptions: NetworkRecordOptions = { - initiatorType: [ + initiatorTypes: [ 'audio', 'beacon', 'body', @@ -151,7 +151,7 @@ function initXhrObserver( win: IWindow, options: Required, ): listenerHandler { - if (!options.initiatorType.includes('xmlhttprequest')) { + if (!options.initiatorTypes.includes('xmlhttprequest')) { return () => { // }; @@ -166,7 +166,7 @@ function initFetchObserver( win: IWindow, options: Required, ): listenerHandler { - if (!options.initiatorType.includes('fetch')) { + if (!options.initiatorTypes.includes('fetch')) { return () => { // }; From a503ca473da3ce69f837ae3d842d51cfac699fb0 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 19:07:57 +0000 Subject: [PATCH 16/43] fix build and rename prop --- packages/rrweb/src/plugins/network/record/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 72c163e2b1..6b80dc224c 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -42,7 +42,7 @@ type NetworkRecordOptions = { request: boolean | StringifyOptions; response: boolean | StringifyOptions; }; - recordInitialEvents?: boolean; + recordInitialRequests?: boolean; }; const defaultNetworkOptions: NetworkRecordOptions = { @@ -69,10 +69,10 @@ const defaultNetworkOptions: NetworkRecordOptions = { 'video', 'xmlhttprequest', ], - ignoreEventFn: () => false, + ignoreRequestFn: () => false, recordHeaders: false, recordBody: false, - recordInitialEvents: false, + recordInitialRequests: false, }; type Headers = Record; @@ -119,7 +119,7 @@ function initPerformanceObserver( ); }); }; - if (options.recordInitialEvents) { + if (options.recordInitialRequests) { const initialPerformanceEntries = getPerformanceEntries( win.performance.getEntries(), ); From ca21babf1b4df30113385e85fe04ed71700e629f Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 19:10:38 +0000 Subject: [PATCH 17/43] bug fix --- packages/rrweb/src/plugins/network/record/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 6b80dc224c..d1bac2edfd 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -290,8 +290,8 @@ function initNetworkObserver( : defaultNetworkOptions) as Required; const cb: networkCallback = (data) => { - const requests = data.requests.filter((request) => - networkOptions.ignoreRequestFn(request), + const requests = data.requests.filter( + (request) => !networkOptions.ignoreRequestFn(request), ); callback({ ...data, requests }); }; From 70395af0a3df354c1b8c32792f78067fb0573832 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 19:39:37 +0000 Subject: [PATCH 18/43] move win.perf check up --- packages/rrweb/src/plugins/network/record/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index d1bac2edfd..8fab9ffd0c 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -98,11 +98,6 @@ function initPerformanceObserver( win: IWindow, options: Required, ) { - if (!('performance' in win)) { - return () => { - // - }; - } const isNavigationTiming = ( entry: PerformanceEntry, ): entry is PerformanceNavigationTiming => entry.entryType === 'navigation'; @@ -285,6 +280,12 @@ function initNetworkObserver( win: IWindow, // top window or in an iframe options: NetworkRecordOptions, ): listenerHandler { + if (!('performance' in win)) { + return () => { + // + }; + } + const networkOptions = (options ? Object.assign({}, defaultNetworkOptions, options) : defaultNetworkOptions) as Required; From 0e08bc7ee6a8929b943e39f0c2a7bd98b5c69f9a Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 21:37:07 +0000 Subject: [PATCH 19/43] better last perf entry --- .../rrweb/src/plugins/network/record/index.ts | 64 +++++++++++-------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 8fab9ffd0c..93efde34c7 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -93,27 +93,42 @@ export type NetworkData = { type networkCallback = (data: NetworkData) => void; +const isNavigationTiming = ( + entry: PerformanceEntry, +): entry is PerformanceNavigationTiming => entry.entryType === 'navigation'; +const isResourceTiming = ( + entry: PerformanceEntry, +): entry is PerformanceResourceTiming => entry.entryType === 'resource'; +const getPerformanceEntries = (entries: PerformanceEntryList) => { + return entries.filter((entry) => { + return ( + isNavigationTiming(entry) || + (isResourceTiming(entry) && + entry.initiatorType !== 'xmlhttprequest' && + entry.initiatorType !== 'fetch') + ); + }); +}; +const getLastPerformanceEntry = ( + win: IWindow, + initiatorType: string, + url: string, +) => { + const performanceEntries = win.performance.getEntriesByType('resource'); + return findLast( + performanceEntries, + (performanceEntry) => + isResourceTiming(performanceEntry) && + performanceEntry.initiatorType === initiatorType && + performanceEntry.name === url, + ); +}; + function initPerformanceObserver( cb: networkCallback, win: IWindow, options: Required, ) { - const isNavigationTiming = ( - entry: PerformanceEntry, - ): entry is PerformanceNavigationTiming => entry.entryType === 'navigation'; - const isResourceTiming = ( - entry: PerformanceEntry, - ): entry is PerformanceResourceTiming => entry.entryType === 'resource'; - const getPerformanceEntries = (entries: PerformanceEntryList) => { - return entries.filter((entry) => { - return ( - isNavigationTiming(entry) || - (isResourceTiming(entry) && - entry.initiatorType !== 'xmlhttprequest' && - entry.initiatorType !== 'fetch') - ); - }); - }; if (options.recordInitialRequests) { const initialPerformanceEntries = getPerformanceEntries( win.performance.getEntries(), @@ -188,8 +203,8 @@ function initFetchObserver( options.recordBody.response); const originalFetch = win.fetch; - const wrappedFetch: typeof fetch = async function (url, init) { - let performanceEntry: PerformanceResourceTiming | undefined; + const wrappedFetch: typeof fetch = async (url, init) => { + let performanceEntry: PerformanceEntry | undefined; const networkRequest: Partial = {}; const req = new Request(url, init); try { @@ -212,13 +227,7 @@ function initFetchObserver( } } const res = await originalFetch(req); - const performanceEntries = win.performance.getEntriesByType( - 'resource', - ) as PerformanceResourceTiming[]; - performanceEntry = findLast( - performanceEntries, - (p) => p.initiatorType === 'fetch' && p.name === req.url, - ); + performanceEntry = getLastPerformanceEntry(win, 'fetch', req.url); if (recordResponseHeaders) { networkRequest.responseHeaders = {}; res.headers.forEach((value, key) => { @@ -244,9 +253,10 @@ function initFetchObserver( } } return res; - // eslint-disable-next-line no-useless-catch } catch (cause) { - // failed to fetch + if (!performanceEntry) { + performanceEntry = getLastPerformanceEntry(win, 'fetch', req.url); + } throw cause; } finally { if (performanceEntry) { From 9d33b47cfeb08f70bbe016821664431148da29a9 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 22:33:07 +0000 Subject: [PATCH 20/43] implement xhr --- .../rrweb/src/plugins/network/record/index.ts | 155 +++++++++++++++++- 1 file changed, 149 insertions(+), 6 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 93efde34c7..dc8e1f35ee 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,5 +1,5 @@ import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; -import { findLast } from '../../../utils'; +import { findLast, patch } from '../../../utils'; import { stringify, StringifyOptions } from '../../utils/stringify'; export type InitiatorType = @@ -166,8 +166,151 @@ function initXhrObserver( // }; } + const recordRequestHeaders = + !!options.recordHeaders && + (typeof options.recordHeaders === 'boolean' || + !('request' in options.recordHeaders) || + options.recordHeaders.request); + const recordRequestBody = + !!options.recordBody && + (typeof options.recordBody === 'boolean' || + !('request' in options.recordBody) || + options.recordBody.request); + const recordResponseHeaders = + !!options.recordHeaders && + (typeof options.recordHeaders === 'boolean' || + !('response' in options.recordHeaders) || + options.recordHeaders.response); + const recordResponseBody = + !!options.recordBody && + (typeof options.recordBody === 'boolean' || + !('response' in options.recordBody) || + options.recordBody.response); + + const restorePatch = patch( + XMLHttpRequest.prototype, + 'open', + (originalOpen: typeof XMLHttpRequest.prototype.open) => { + return async function ( + method: string, + url: string | URL, + async = true, + username?: string | null, + password?: string | null, + ) { + let performanceEntry: PerformanceEntry | undefined; + const networkRequest: Partial = {}; + const xhr = this as XMLHttpRequest; + const requestUrl = typeof url === 'string' ? url : url.toString(); + try { + if (recordRequestHeaders) { + networkRequest.requestHeaders = {}; + const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); + xhr.setRequestHeader = (header: string, value: string) => { + networkRequest.requestHeaders![header] = value; + return originalSetRequestHeader(header, value); + }; + } + if (recordRequestBody) { + const originalSend = xhr.send.bind(xhr); + xhr.send = (body) => { + if (!body) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = stringify( + body, + typeof recordRequestBody === 'object' + ? recordRequestBody + : undefined, + ); + } + return originalSend(body); + }; + } + await new Promise((resolve, reject) => { + try { + if (recordResponseBody) { + xhr.addEventListener('load', () => { + try { + if (!xhr.response) { + networkRequest.responseBody = null; + } else { + try { + const objBody = JSON.parse( + xhr.response as string, + ) as object; + networkRequest.responseBody = stringify( + objBody, + typeof recordResponseBody === 'object' + ? recordResponseBody + : undefined, + ); + } catch { + networkRequest.responseBody = xhr.response as string; + } + } + } catch (cause) { + reject(cause); + } + }); + } + xhr.addEventListener('loadend', () => { + try { + performanceEntry = getLastPerformanceEntry( + win, + 'xmlhttprequest', + requestUrl, + ); + if (recordResponseHeaders) { + networkRequest.responseHeaders = {}; + const rawHeaders = xhr.getAllResponseHeaders(); + const headers = rawHeaders.trim().split(/[\r\n]+/); + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (header) { + networkRequest.responseHeaders![header] = value; + } + }); + } + resolve(); + } catch (cause) { + reject(cause); + } + }); + originalOpen(method, url, async, username, password); + } catch (cause) { + reject(cause); + } + }); + } catch (cause) { + if (!performanceEntry) { + performanceEntry = getLastPerformanceEntry( + win, + 'xmlhttprequest', + requestUrl, + ); + } + throw cause; + } finally { + if (performanceEntry) { + cb({ + requests: [ + { + performanceEntry, + requestMethod: method, + ...networkRequest, + }, + ], + }); + } + } + }; + }, + ); return () => { - // TODO: + restorePatch(); }; } @@ -210,8 +353,8 @@ function initFetchObserver( try { if (recordRequestHeaders) { networkRequest.requestHeaders = {}; - req.headers.forEach((value, key) => { - networkRequest.requestHeaders![key] = value; + req.headers.forEach((value, header) => { + networkRequest.requestHeaders![header] = value; }); } if (recordRequestBody) { @@ -230,8 +373,8 @@ function initFetchObserver( performanceEntry = getLastPerformanceEntry(win, 'fetch', req.url); if (recordResponseHeaders) { networkRequest.responseHeaders = {}; - res.headers.forEach((value, key) => { - networkRequest.responseHeaders![key] = value; + res.headers.forEach((value, header) => { + networkRequest.responseHeaders![header] = value; }); } if (recordResponseBody) { From a17496481f4ac1ce1d31295b212f8b891cba63d4 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 22:54:59 +0000 Subject: [PATCH 21/43] better xhr events implementation --- .../rrweb/src/plugins/network/record/index.ts | 155 ++++++++---------- 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index dc8e1f35ee..6de0a7ad53 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -99,36 +99,22 @@ const isNavigationTiming = ( const isResourceTiming = ( entry: PerformanceEntry, ): entry is PerformanceResourceTiming => entry.entryType === 'resource'; -const getPerformanceEntries = (entries: PerformanceEntryList) => { - return entries.filter((entry) => { - return ( - isNavigationTiming(entry) || - (isResourceTiming(entry) && - entry.initiatorType !== 'xmlhttprequest' && - entry.initiatorType !== 'fetch') - ); - }); -}; -const getLastPerformanceEntry = ( - win: IWindow, - initiatorType: string, - url: string, -) => { - const performanceEntries = win.performance.getEntriesByType('resource'); - return findLast( - performanceEntries, - (performanceEntry) => - isResourceTiming(performanceEntry) && - performanceEntry.initiatorType === initiatorType && - performanceEntry.name === url, - ); -}; function initPerformanceObserver( cb: networkCallback, win: IWindow, options: Required, ) { + const getPerformanceEntries = (entries: PerformanceEntryList) => { + return entries.filter((entry) => { + return ( + isNavigationTiming(entry) || + (isResourceTiming(entry) && + entry.initiatorType !== 'xmlhttprequest' && // ignore xhr + entry.initiatorType !== 'fetch') // ignore fetch + ); + }); + }; if (options.recordInitialRequests) { const initialPerformanceEntries = getPerformanceEntries( win.performance.getEntries(), @@ -156,6 +142,21 @@ function initPerformanceObserver( }; } +const getPerformanceEntry = ( + win: IWindow, + initiatorType: string, + url: string, +) => { + const performanceEntries = win.performance.getEntriesByType('resource'); + return findLast( + performanceEntries, + (performanceEntry) => + isResourceTiming(performanceEntry) && + performanceEntry.initiatorType === initiatorType && + performanceEntry.name === url, + ); +}; + function initXhrObserver( cb: networkCallback, win: IWindow, @@ -198,10 +199,10 @@ function initXhrObserver( username?: string | null, password?: string | null, ) { - let performanceEntry: PerformanceEntry | undefined; - const networkRequest: Partial = {}; const xhr = this as XMLHttpRequest; const requestUrl = typeof url === 'string' ? url : url.toString(); + let performanceEntry: PerformanceEntry | undefined; + const networkRequest: Partial = {}; try { if (recordRequestHeaders) { networkRequest.requestHeaders = {}; @@ -214,7 +215,7 @@ function initXhrObserver( if (recordRequestBody) { const originalSend = xhr.send.bind(xhr); xhr.send = (body) => { - if (!body) { + if (body === undefined || body === null) { networkRequest.requestBody = null; } else { networkRequest.requestBody = stringify( @@ -227,66 +228,54 @@ function initXhrObserver( return originalSend(body); }; } - await new Promise((resolve, reject) => { - try { - if (recordResponseBody) { - xhr.addEventListener('load', () => { - try { - if (!xhr.response) { - networkRequest.responseBody = null; - } else { - try { - const objBody = JSON.parse( - xhr.response as string, - ) as object; - networkRequest.responseBody = stringify( - objBody, - typeof recordResponseBody === 'object' - ? recordResponseBody - : undefined, - ); - } catch { - networkRequest.responseBody = xhr.response as string; - } - } - } catch (cause) { - reject(cause); + await new Promise((resolve) => { + xhr.responseType = 'text'; + xhr.addEventListener('readystatechange', () => { + if (xhr.readyState !== xhr.DONE) return; + performanceEntry = getPerformanceEntry( + win, + 'xmlhttprequest', + requestUrl, + ); + if (recordResponseHeaders) { + networkRequest.responseHeaders = {}; + const rawHeaders = xhr.getAllResponseHeaders(); + const headers = rawHeaders.trim().split(/[\r\n]+/); + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (header) { + networkRequest.responseHeaders![header] = value; } }); } - xhr.addEventListener('loadend', () => { - try { - performanceEntry = getLastPerformanceEntry( - win, - 'xmlhttprequest', - requestUrl, - ); - if (recordResponseHeaders) { - networkRequest.responseHeaders = {}; - const rawHeaders = xhr.getAllResponseHeaders(); - const headers = rawHeaders.trim().split(/[\r\n]+/); - headers.forEach((line) => { - const parts = line.split(': '); - const header = parts.shift(); - const value = parts.join(': '); - if (header) { - networkRequest.responseHeaders![header] = value; - } - }); + if (recordResponseBody) { + if (!xhr.response) { + networkRequest.responseBody = null; + } else { + try { + const objBody = JSON.parse( + xhr.response as string, + ) as object; + networkRequest.responseBody = stringify( + objBody, + typeof recordResponseBody === 'object' + ? recordResponseBody + : undefined, + ); + } catch { + networkRequest.responseBody = xhr.response as string; } - resolve(); - } catch (cause) { - reject(cause); } - }); - originalOpen(method, url, async, username, password); - } catch (cause) { - reject(cause); - } + } + resolve(); + }); + originalOpen(method, url, async, username, password); }); } catch (cause) { if (!performanceEntry) { - performanceEntry = getLastPerformanceEntry( + performanceEntry = getPerformanceEntry( win, 'xmlhttprequest', requestUrl, @@ -347,9 +336,9 @@ function initFetchObserver( const originalFetch = win.fetch; const wrappedFetch: typeof fetch = async (url, init) => { + const req = new Request(url, init); let performanceEntry: PerformanceEntry | undefined; const networkRequest: Partial = {}; - const req = new Request(url, init); try { if (recordRequestHeaders) { networkRequest.requestHeaders = {}; @@ -358,7 +347,7 @@ function initFetchObserver( }); } if (recordRequestBody) { - if (!req.body) { + if (req.body === undefined || req.body === null) { networkRequest.requestBody = null; } else { networkRequest.requestBody = stringify( @@ -370,7 +359,7 @@ function initFetchObserver( } } const res = await originalFetch(req); - performanceEntry = getLastPerformanceEntry(win, 'fetch', req.url); + performanceEntry = getPerformanceEntry(win, 'fetch', req.url); if (recordResponseHeaders) { networkRequest.responseHeaders = {}; res.headers.forEach((value, header) => { @@ -398,7 +387,7 @@ function initFetchObserver( return res; } catch (cause) { if (!performanceEntry) { - performanceEntry = getLastPerformanceEntry(win, 'fetch', req.url); + performanceEntry = getPerformanceEntry(win, 'fetch', req.url); } throw cause; } finally { From 7f9ba0d7a1791e0170e7abd7d4369a182fce28f3 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 23:03:31 +0000 Subject: [PATCH 22/43] skip empty request callbacks --- packages/rrweb/src/plugins/network/record/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 6de0a7ad53..0e056eec56 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -148,6 +148,11 @@ const getPerformanceEntry = ( url: string, ) => { const performanceEntries = win.performance.getEntriesByType('resource'); + console.log('getPerformanceEntry', { + performanceEntries, + initiatorType, + url, + }); return findLast( performanceEntries, (performanceEntry) => @@ -436,7 +441,9 @@ function initNetworkObserver( const requests = data.requests.filter( (request) => !networkOptions.ignoreRequestFn(request), ); - callback({ ...data, requests }); + if (requests.length > 0 || data.isInitial) { + callback({ ...data, requests }); + } }; const performanceObserver = initPerformanceObserver(cb, win, networkOptions); From ad966bcb0567ed9bd2cbdae4f029eeaa24c7b023 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 23:45:44 +0000 Subject: [PATCH 23/43] by url --- .../rrweb/src/plugins/network/record/index.ts | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 0e056eec56..8be278897b 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -142,23 +142,17 @@ function initPerformanceObserver( }; } -const getPerformanceEntry = ( +const getPerformanceEntryByUrl = ( win: IWindow, initiatorType: string, url: string, ) => { - const performanceEntries = win.performance.getEntriesByType('resource'); - console.log('getPerformanceEntry', { - performanceEntries, - initiatorType, - url, - }); + const urlPerformanceEntries = win.performance.getEntriesByName(url); return findLast( - performanceEntries, + urlPerformanceEntries, (performanceEntry) => isResourceTiming(performanceEntry) && - performanceEntry.initiatorType === initiatorType && - performanceEntry.name === url, + performanceEntry.initiatorType === initiatorType, ); }; @@ -236,8 +230,10 @@ function initXhrObserver( await new Promise((resolve) => { xhr.responseType = 'text'; xhr.addEventListener('readystatechange', () => { - if (xhr.readyState !== xhr.DONE) return; - performanceEntry = getPerformanceEntry( + if (xhr.readyState !== xhr.DONE) { + return; + } + performanceEntry = getPerformanceEntryByUrl( win, 'xmlhttprequest', requestUrl, @@ -280,7 +276,7 @@ function initXhrObserver( }); } catch (cause) { if (!performanceEntry) { - performanceEntry = getPerformanceEntry( + performanceEntry = getPerformanceEntryByUrl( win, 'xmlhttprequest', requestUrl, @@ -364,7 +360,7 @@ function initFetchObserver( } } const res = await originalFetch(req); - performanceEntry = getPerformanceEntry(win, 'fetch', req.url); + performanceEntry = getPerformanceEntryByUrl(win, 'fetch', req.url); if (recordResponseHeaders) { networkRequest.responseHeaders = {}; res.headers.forEach((value, header) => { @@ -392,7 +388,7 @@ function initFetchObserver( return res; } catch (cause) { if (!performanceEntry) { - performanceEntry = getPerformanceEntry(win, 'fetch', req.url); + performanceEntry = getPerformanceEntryByUrl(win, 'fetch', req.url); } throw cause; } finally { From 0ae1f060734afb79e22891586dead4b04c2e42a6 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 1 Feb 2023 23:46:27 +0000 Subject: [PATCH 24/43] linting --- packages/rrweb/src/plugins/network/record/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 8be278897b..424b44614a 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -428,7 +428,6 @@ function initNetworkObserver( // }; } - const networkOptions = (options ? Object.assign({}, defaultNetworkOptions, options) : defaultNetworkOptions) as Required; @@ -441,11 +440,9 @@ function initNetworkObserver( callback({ ...data, requests }); } }; - const performanceObserver = initPerformanceObserver(cb, win, networkOptions); const xhrObserver = initXhrObserver(cb, win, networkOptions); const fetchObserver = initFetchObserver(cb, win, networkOptions); - return () => { performanceObserver(); xhrObserver(); From 147aba645d9c0f42d23ff18e42a531b03cef5fc6 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 00:08:11 +0000 Subject: [PATCH 25/43] fix xhr --- .../rrweb/src/plugins/network/record/index.ts | 163 ++++++++---------- 1 file changed, 74 insertions(+), 89 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 424b44614a..1fa7ed720c 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -199,102 +199,87 @@ function initXhrObserver( password?: string | null, ) { const xhr = this as XMLHttpRequest; - const requestUrl = typeof url === 'string' ? url : url.toString(); + const req = new Request(url); let performanceEntry: PerformanceEntry | undefined; const networkRequest: Partial = {}; - try { - if (recordRequestHeaders) { - networkRequest.requestHeaders = {}; - const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); - xhr.setRequestHeader = (header: string, value: string) => { - networkRequest.requestHeaders![header] = value; - return originalSetRequestHeader(header, value); - }; - } - if (recordRequestBody) { - const originalSend = xhr.send.bind(xhr); - xhr.send = (body) => { - if (body === undefined || body === null) { - networkRequest.requestBody = null; - } else { - networkRequest.requestBody = stringify( - body, - typeof recordRequestBody === 'object' - ? recordRequestBody - : undefined, - ); - } - return originalSend(body); - }; - } - await new Promise((resolve) => { - xhr.responseType = 'text'; - xhr.addEventListener('readystatechange', () => { - if (xhr.readyState !== xhr.DONE) { - return; - } - performanceEntry = getPerformanceEntryByUrl( - win, - 'xmlhttprequest', - requestUrl, + if (recordRequestHeaders) { + networkRequest.requestHeaders = {}; + const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); + xhr.setRequestHeader = (header: string, value: string) => { + networkRequest.requestHeaders![header] = value; + return originalSetRequestHeader(header, value); + }; + } + if (recordRequestBody) { + const originalSend = xhr.send.bind(xhr); + xhr.send = (body) => { + if (body === undefined || body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = stringify( + body, + typeof recordRequestBody === 'object' + ? recordRequestBody + : undefined, ); - if (recordResponseHeaders) { - networkRequest.responseHeaders = {}; - const rawHeaders = xhr.getAllResponseHeaders(); - const headers = rawHeaders.trim().split(/[\r\n]+/); - headers.forEach((line) => { - const parts = line.split(': '); - const header = parts.shift(); - const value = parts.join(': '); - if (header) { - networkRequest.responseHeaders![header] = value; - } - }); - } - if (recordResponseBody) { - if (!xhr.response) { - networkRequest.responseBody = null; - } else { - try { - const objBody = JSON.parse( - xhr.response as string, - ) as object; - networkRequest.responseBody = stringify( - objBody, - typeof recordResponseBody === 'object' - ? recordResponseBody - : undefined, - ); - } catch { - networkRequest.responseBody = xhr.response as string; - } - } - } - resolve(); - }); - originalOpen(method, url, async, username, password); - }); - } catch (cause) { - if (!performanceEntry) { + } + return originalSend(body); + }; + } + await new Promise((resolve) => { + xhr.addEventListener('readystatechange', () => { + if (xhr.readyState !== xhr.DONE) { + return; + } performanceEntry = getPerformanceEntryByUrl( win, 'xmlhttprequest', - requestUrl, + req.url, ); - } - throw cause; - } finally { - if (performanceEntry) { - cb({ - requests: [ - { - performanceEntry, - requestMethod: method, - ...networkRequest, - }, - ], - }); - } + if (recordResponseHeaders) { + networkRequest.responseHeaders = {}; + const rawHeaders = xhr.getAllResponseHeaders(); + const headers = rawHeaders.trim().split(/[\r\n]+/); + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (header) { + networkRequest.responseHeaders![header] = value; + } + }); + } + if (recordResponseBody) { + if (!xhr.response) { + networkRequest.responseBody = null; + } else { + try { + const objBody = JSON.parse(xhr.response as string) as object; + networkRequest.responseBody = stringify( + objBody, + typeof recordResponseBody === 'object' + ? recordResponseBody + : undefined, + ); + } catch { + networkRequest.responseBody = xhr.response as string; + } + } + } + resolve(); + }); + originalOpen(method, url, async, username, password); + }); + if (performanceEntry) { + cb({ + requests: [ + { + performanceEntry, + requestMethod: req.method, + ...networkRequest, + }, + ], + }); } }; }, From ec4010f3f6e9faf8e8264db0cb155fdf0ffe01f6 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 02:49:05 +0000 Subject: [PATCH 26/43] better getRequestPerformanceEntry --- .../rrweb/src/plugins/network/record/index.ts | 178 ++++++++++-------- 1 file changed, 99 insertions(+), 79 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 1fa7ed720c..4bd2556ba9 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -142,18 +142,38 @@ function initPerformanceObserver( }; } -const getPerformanceEntryByUrl = ( +const getRequestPerformanceEntry = async ( win: IWindow, initiatorType: string, url: string, -) => { + after?: number, + before?: number, + attempt = 0, +): Promise => { + if (attempt > 10) { + throw new Error('Cannot find performance entry'); + } const urlPerformanceEntries = win.performance.getEntriesByName(url); - return findLast( + const performanceEntry = findLast( urlPerformanceEntries, (performanceEntry) => isResourceTiming(performanceEntry) && - performanceEntry.initiatorType === initiatorType, + performanceEntry.initiatorType === initiatorType && + (!after || performanceEntry.startTime >= after) && + (!before || performanceEntry.startTime <= before), ); + if (!performanceEntry) { + await new Promise((resolve) => setTimeout(resolve, 50 * attempt)); + return getRequestPerformanceEntry( + win, + initiatorType, + url, + after, + before, + attempt + 1, + ); + } + return performanceEntry; }; function initXhrObserver( @@ -191,7 +211,7 @@ function initXhrObserver( XMLHttpRequest.prototype, 'open', (originalOpen: typeof XMLHttpRequest.prototype.open) => { - return async function ( + return function ( method: string, url: string | URL, async = true, @@ -200,8 +220,9 @@ function initXhrObserver( ) { const xhr = this as XMLHttpRequest; const req = new Request(url); - let performanceEntry: PerformanceEntry | undefined; const networkRequest: Partial = {}; + let after: number | undefined; + let before: number | undefined; if (recordRequestHeaders) { networkRequest.requestHeaders = {}; const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); @@ -210,9 +231,9 @@ function initXhrObserver( return originalSetRequestHeader(header, value); }; } - if (recordRequestBody) { - const originalSend = xhr.send.bind(xhr); - xhr.send = (body) => { + const originalSend = xhr.send.bind(xhr); + xhr.send = (body) => { + if (recordRequestBody) { if (body === undefined || body === null) { networkRequest.requestBody = null; } else { @@ -223,64 +244,65 @@ function initXhrObserver( : undefined, ); } - return originalSend(body); - }; - } - await new Promise((resolve) => { - xhr.addEventListener('readystatechange', () => { - if (xhr.readyState !== xhr.DONE) { - return; - } - performanceEntry = getPerformanceEntryByUrl( - win, - 'xmlhttprequest', - req.url, - ); - if (recordResponseHeaders) { - networkRequest.responseHeaders = {}; - const rawHeaders = xhr.getAllResponseHeaders(); - const headers = rawHeaders.trim().split(/[\r\n]+/); - headers.forEach((line) => { - const parts = line.split(': '); - const header = parts.shift(); - const value = parts.join(': '); - if (header) { - networkRequest.responseHeaders![header] = value; - } - }); - } - if (recordResponseBody) { - if (!xhr.response) { - networkRequest.responseBody = null; - } else { - try { - const objBody = JSON.parse(xhr.response as string) as object; - networkRequest.responseBody = stringify( - objBody, - typeof recordResponseBody === 'object' - ? recordResponseBody - : undefined, - ); - } catch { - networkRequest.responseBody = xhr.response as string; - } + } + after = win.performance.now(); + return originalSend(body); + }; + xhr.addEventListener('readystatechange', () => { + if (xhr.readyState !== xhr.DONE) { + return; + } + before = win.performance.now(); + if (recordResponseHeaders) { + networkRequest.responseHeaders = {}; + const rawHeaders = xhr.getAllResponseHeaders(); + const headers = rawHeaders.trim().split(/[\r\n]+/); + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (header) { + networkRequest.responseHeaders![header] = value; + } + }); + } + if (recordResponseBody) { + if (!xhr.response) { + networkRequest.responseBody = null; + } else { + try { + const objBody = JSON.parse(xhr.response as string) as object; + networkRequest.responseBody = stringify( + objBody, + typeof recordResponseBody === 'object' + ? recordResponseBody + : undefined, + ); + } catch { + networkRequest.responseBody = xhr.response as string; } } - resolve(); - }); - originalOpen(method, url, async, username, password); - }); - if (performanceEntry) { - cb({ - requests: [ - { + } + getRequestPerformanceEntry( + win, + 'xmlhttprequest', + req.url, + after, + before, + ) + .then((performanceEntry) => { + const request: NetworkRequest = { performanceEntry, requestMethod: req.method, ...networkRequest, - }, - ], - }); - } + }; + cb({ requests: [request] }); + }) + .catch(() => { + // + }); + }); + originalOpen(method, url, async, username, password); }; }, ); @@ -323,8 +345,9 @@ function initFetchObserver( const originalFetch = win.fetch; const wrappedFetch: typeof fetch = async (url, init) => { const req = new Request(url, init); - let performanceEntry: PerformanceEntry | undefined; const networkRequest: Partial = {}; + let after: number | undefined; + let before: number | undefined; try { if (recordRequestHeaders) { networkRequest.requestHeaders = {}; @@ -344,8 +367,9 @@ function initFetchObserver( ); } } + after = win.performance.now(); const res = await originalFetch(req); - performanceEntry = getPerformanceEntryByUrl(win, 'fetch', req.url); + before = win.performance.now(); if (recordResponseHeaders) { networkRequest.responseHeaders = {}; res.headers.forEach((value, header) => { @@ -371,23 +395,19 @@ function initFetchObserver( } } return res; - } catch (cause) { - if (!performanceEntry) { - performanceEntry = getPerformanceEntryByUrl(win, 'fetch', req.url); - } - throw cause; } finally { - if (performanceEntry) { - cb({ - requests: [ - { - performanceEntry, - requestMethod: req.method, - ...networkRequest, - }, - ], + getRequestPerformanceEntry(win, 'fetch', req.url, after, before) + .then((performanceEntry) => { + const request: NetworkRequest = { + performanceEntry, + requestMethod: req.method, + ...networkRequest, + }; + cb({ requests: [request] }); + }) + .catch(() => { + // }); - } } }; wrappedFetch.prototype = {}; From c5a1d1466fca6f7a1346eae100de75842f976780 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 03:11:16 +0000 Subject: [PATCH 27/43] dont filter xhr & fetch from initial --- .../rrweb/src/plugins/network/record/index.ts | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 4bd2556ba9..8ff3c58213 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -105,20 +105,17 @@ function initPerformanceObserver( win: IWindow, options: Required, ) { - const getPerformanceEntries = (entries: PerformanceEntryList) => { - return entries.filter((entry) => { - return ( - isNavigationTiming(entry) || - (isResourceTiming(entry) && - entry.initiatorType !== 'xmlhttprequest' && // ignore xhr - entry.initiatorType !== 'fetch') // ignore fetch - ); - }); - }; if (options.recordInitialRequests) { - const initialPerformanceEntries = getPerformanceEntries( - win.performance.getEntries(), - ); + const initialPerformanceEntries = win.performance + .getEntries() + .filter( + (entry) => + isNavigationTiming(entry) || + (isResourceTiming(entry) && + options.initiatorTypes.includes( + entry.initiatorType as InitiatorType, + )), + ); cb({ requests: initialPerformanceEntries.map((performanceEntry) => ({ performanceEntry, @@ -128,7 +125,18 @@ function initPerformanceObserver( }); } const observer = new win.PerformanceObserver((entries) => { - const performanceEntries = getPerformanceEntries(entries.getEntries()); + const performanceEntries = entries + .getEntries() + .filter( + (entry) => + isNavigationTiming(entry) || + (isResourceTiming(entry) && + options.initiatorTypes.includes( + entry.initiatorType as InitiatorType, + ) && + entry.initiatorType !== 'xmlhttprequest' && + entry.initiatorType !== 'fetch'), + ); cb({ requests: performanceEntries.map((performanceEntry) => ({ performanceEntry, From 65662948e802b4ee55c883b1e05c0cf5f08c21d7 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 03:46:01 +0000 Subject: [PATCH 28/43] fix xhr --- packages/rrweb/src/plugins/network/record/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 8ff3c58213..3f049f55f5 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -310,7 +310,7 @@ function initXhrObserver( // }); }); - originalOpen(method, url, async, username, password); + originalOpen.call(xhr, method, url, async, username, password); }; }, ); From dcf372151ac414b4ab3ebd7cf35193a6ce5046d7 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 04:19:48 +0000 Subject: [PATCH 29/43] add responseStatus --- packages/rrweb/src/plugins/network/record/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 3f049f55f5..fd5e2e7f9a 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -82,6 +82,7 @@ type NetworkRequest = { requestMethod: string; requestHeaders?: Headers; requestBody?: string | null; + responseStatus?: number; responseHeaders?: Headers; responseBody?: string | null; }; @@ -302,6 +303,7 @@ function initXhrObserver( const request: NetworkRequest = { performanceEntry, requestMethod: req.method, + responseStatus: xhr.status, ...networkRequest, }; cb({ requests: [request] }); @@ -378,6 +380,7 @@ function initFetchObserver( after = win.performance.now(); const res = await originalFetch(req); before = win.performance.now(); + networkRequest.responseStatus = res.status; if (recordResponseHeaders) { networkRequest.responseHeaders = {}; res.headers.forEach((value, header) => { From a131e37bb82678b30e35303fe425150439b6f1d5 Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 04:24:30 +0000 Subject: [PATCH 30/43] docs --- docs/recipes/network.md | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 docs/recipes/network.md diff --git a/docs/recipes/network.md b/docs/recipes/network.md new file mode 100644 index 0000000000..45892cc1e8 --- /dev/null +++ b/docs/recipes/network.md @@ -0,0 +1,97 @@ +# network recorder and replayer + +Starting from v2.0.0, we add the plugin to record network output. +This feature aims to provide developers with more information about the bug scene. There are some options for recording and replaying network output. + +### Enable recording network + +You can enable using default option like this: + +```js +rrweb.record({ + emit: function emit(event) { + events.push(event); + }, + // to use default record option + plugins: [rrweb.getRecordNetworkPlugin()], +}); +``` + +You can also customize the behavior of logger like this: + +```js +rrweb.record({ + emit: function emit(event) { + fetch('https://api.my-server.com/events', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + events: [event], + }), + }); + }, + // customized record options + plugins: [ + rrweb.getRecordConsolePlugin({ + initiatorTypes: ['fetch', 'xmlhttprequest'], + // block recording event for request to upload events to server + ignoreRequestFn: ({ performanceEntry }) => { + if (performanceEntry.name === 'https://api.my-server.com/events') { + return true; + } + return false; + }, + recordHeaders: true, + recordBody: true, + recordInitialRequests: false, + }), + ], +}); +``` + +**alert**: If you are uploading events to a server, you should always use `ignoreRequestFn` to block recording events for these requests or else you will cause a nasty loop. + +All options are described below: +| key | default | description | +| ---------------- | --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| initiatorTypes | ['fetch','xmlhttprequest','img',...] | Default value contains names of all [initiator types](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming/initiatorType). You can override it by setting the types you need. | +| ignoreRequestFn | () => false | Block recording events for specific requests | +| recordHeaders | false | Record the request & response headers for `fetch` and `xmlhttprequest` requests | +| recordBody | false | Record the request & response bodies for `fetch` and `xmlhttprequest` requests | +| recordInitialRequests | false | Record an event for all requests prior to rrweb.record() being called | + +## replay network + +It is up to you to decide how to best replay your network events using the `onNetworkData` callback. + +```js +const replayer = new rrweb.Replayer(events, { + plugins: [ + rrweb.getReplayNetworkPlugin({ + onNetworkData: ({ requests }) => { + for (const request of requests) { + const url = request.performanceEntry.name; + const method = request.responseMethod; + const status = request.responseStatus; + console.log(`${method} ${url} ${status}`); + } + }, + }), + ], +}); +replayer.play(); +``` + +Description of replay option is as follows: + +| key | default | description | +| ------------- | --------- | ------------------------------------------------------------------------------------------ | +| onNetworkData | undefined | You could use this interface to replay the network requests in a simulated browser console | + +## technical implementation + +This implementation records [`fetch`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) and [`XMLHttpRequest`](https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest) by patching their object & methods. We record document navigation using [`PerformanceNavigationTiming`](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceNavigationTiming) and we use [`PerformanceResourceTiming`](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming) for recording everything else (script, img, link etc.) via [`PerformanceObserver`](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceObserver) API. + +For more information please see [[network-plugin] Feat: Capture network events #1105](https://github.com/rrweb-io/rrweb/pull/1105) PR. From dba49bf4ea31d878d141d65f091446626879435f Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 2 Feb 2023 04:30:33 +0000 Subject: [PATCH 31/43] formatting --- docs/recipes/network.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/recipes/network.md b/docs/recipes/network.md index 45892cc1e8..1d4f165b21 100644 --- a/docs/recipes/network.md +++ b/docs/recipes/network.md @@ -56,11 +56,11 @@ rrweb.record({ All options are described below: | key | default | description | | ---------------- | --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| initiatorTypes | ['fetch','xmlhttprequest','img',...] | Default value contains names of all [initiator types](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming/initiatorType). You can override it by setting the types you need. | -| ignoreRequestFn | () => false | Block recording events for specific requests | -| recordHeaders | false | Record the request & response headers for `fetch` and `xmlhttprequest` requests | -| recordBody | false | Record the request & response bodies for `fetch` and `xmlhttprequest` requests | -| recordInitialRequests | false | Record an event for all requests prior to rrweb.record() being called | +| initiatorTypes | `['fetch','xmlhttprequest','img',...]` | Default value contains names of all [initiator types](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming/initiatorType). You can override it by setting the types you need. | +| ignoreRequestFn | `() => false` | Block recording events for specific requests | +| recordHeaders | `false` | Record the request & response headers for `fetch` and `xmlhttprequest` requests | +| recordBody | `false` | Record the request & response bodies for `fetch` and `xmlhttprequest` requests | +| recordInitialRequests | `false` | Record an event for all requests prior to rrweb.record() being called | ## replay network @@ -86,9 +86,9 @@ replayer.play(); Description of replay option is as follows: -| key | default | description | -| ------------- | --------- | ------------------------------------------------------------------------------------------ | -| onNetworkData | undefined | You could use this interface to replay the network requests in a simulated browser console | +| key | default | description | +| ------------- | ----------- | ------------------------------------------------------------------------------------------ | +| onNetworkData | `undefined` | You could use this interface to replay the network requests in a simulated browser console | ## technical implementation From 15ecfcfeb58627560d49d72e8238b6778054a03b Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 4 Feb 2023 11:13:24 +0000 Subject: [PATCH 32/43] slim NetworkRequest --- .../rrweb/src/plugins/network/record/index.ts | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index fd5e2e7f9a..e85cf9d168 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -78,11 +78,13 @@ const defaultNetworkOptions: NetworkRecordOptions = { type Headers = Record; type NetworkRequest = { - performanceEntry: PerformanceEntry; + requestUrl: string; requestMethod: string; + requestInitiator: InitiatorType; requestHeaders?: Headers; requestBody?: string | null; responseStatus?: number; + responseDuration: number; responseHeaders?: Headers; responseBody?: string | null; }; @@ -101,6 +103,13 @@ const isResourceTiming = ( entry: PerformanceEntry, ): entry is PerformanceResourceTiming => entry.entryType === 'resource'; +type ObservedPerformanceEntry = ( + | PerformanceNavigationTiming + | PerformanceResourceTiming +) & { + responseStatus?: number; +}; + function initPerformanceObserver( cb: networkCallback, win: IWindow, @@ -110,7 +119,7 @@ function initPerformanceObserver( const initialPerformanceEntries = win.performance .getEntries() .filter( - (entry) => + (entry): entry is ObservedPerformanceEntry => isNavigationTiming(entry) || (isResourceTiming(entry) && options.initiatorTypes.includes( @@ -119,8 +128,14 @@ function initPerformanceObserver( ); cb({ requests: initialPerformanceEntries.map((performanceEntry) => ({ - performanceEntry, + requestUrl: performanceEntry.name, requestMethod: 'GET', + requestInitiator: performanceEntry.initiatorType as InitiatorType, + responseStatus: + 'responseStatus' in performanceEntry + ? performanceEntry.responseStatus + : undefined, + responseDuration: Math.round(performanceEntry.duration), })), isInitial: true, }); @@ -129,7 +144,7 @@ function initPerformanceObserver( const performanceEntries = entries .getEntries() .filter( - (entry) => + (entry): entry is ObservedPerformanceEntry => isNavigationTiming(entry) || (isResourceTiming(entry) && options.initiatorTypes.includes( @@ -140,8 +155,14 @@ function initPerformanceObserver( ); cb({ requests: performanceEntries.map((performanceEntry) => ({ - performanceEntry, + requestUrl: performanceEntry.name, requestMethod: 'GET', + requestInitiator: performanceEntry.initiatorType as InitiatorType, + responseStatus: + 'responseStatus' in performanceEntry + ? performanceEntry.responseStatus + : undefined, + responseDuration: Math.round(performanceEntry.duration), })), }); }); @@ -158,11 +179,13 @@ const getRequestPerformanceEntry = async ( after?: number, before?: number, attempt = 0, -): Promise => { +): Promise => { if (attempt > 10) { throw new Error('Cannot find performance entry'); } - const urlPerformanceEntries = win.performance.getEntriesByName(url); + const urlPerformanceEntries = win.performance.getEntriesByName( + url, + ) as PerformanceResourceTiming[]; const performanceEntry = findLast( urlPerformanceEntries, (performanceEntry) => @@ -301,10 +324,15 @@ function initXhrObserver( ) .then((performanceEntry) => { const request: NetworkRequest = { - performanceEntry, + requestUrl: performanceEntry.name, requestMethod: req.method, + requestInitiator: performanceEntry.initiatorType as InitiatorType, + requestHeaders: networkRequest.requestHeaders, + requestBody: networkRequest.requestBody, responseStatus: xhr.status, - ...networkRequest, + responseDuration: performanceEntry.duration, + responseHeaders: networkRequest.responseHeaders, + responseBody: networkRequest.responseBody, }; cb({ requests: [request] }); }) @@ -355,6 +383,7 @@ function initFetchObserver( const originalFetch = win.fetch; const wrappedFetch: typeof fetch = async (url, init) => { const req = new Request(url, init); + let res: Response | undefined; const networkRequest: Partial = {}; let after: number | undefined; let before: number | undefined; @@ -378,7 +407,7 @@ function initFetchObserver( } } after = win.performance.now(); - const res = await originalFetch(req); + res = await originalFetch(req); before = win.performance.now(); networkRequest.responseStatus = res.status; if (recordResponseHeaders) { @@ -410,9 +439,15 @@ function initFetchObserver( getRequestPerformanceEntry(win, 'fetch', req.url, after, before) .then((performanceEntry) => { const request: NetworkRequest = { - performanceEntry, + requestUrl: performanceEntry.name, requestMethod: req.method, - ...networkRequest, + requestInitiator: performanceEntry.initiatorType as InitiatorType, + requestHeaders: networkRequest.requestHeaders, + requestBody: networkRequest.requestBody, + responseStatus: res?.status, + responseDuration: performanceEntry.duration, + responseHeaders: networkRequest.responseHeaders, + responseBody: networkRequest.responseBody, }; cb({ requests: [request] }); }) From fc5b77a81db4b1be2b1e340f6f10ab2ddfe43ca4 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 4 Feb 2023 11:16:10 +0000 Subject: [PATCH 33/43] update docs --- docs/recipes/network.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recipes/network.md b/docs/recipes/network.md index 1d4f165b21..5764872bcd 100644 --- a/docs/recipes/network.md +++ b/docs/recipes/network.md @@ -72,7 +72,7 @@ const replayer = new rrweb.Replayer(events, { rrweb.getReplayNetworkPlugin({ onNetworkData: ({ requests }) => { for (const request of requests) { - const url = request.performanceEntry.name; + const url = request.requestUrl; const method = request.responseMethod; const status = request.responseStatus; console.log(`${method} ${url} ${status}`); From a0c721be38f306b0c532b45c6d133bbe44924e34 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 12:19:59 +0000 Subject: [PATCH 34/43] startTime & endTime --- .../rrweb/src/plugins/network/record/index.ts | 78 +++++++++---------- 1 file changed, 37 insertions(+), 41 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index e85cf9d168..bd8260848e 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -78,13 +78,14 @@ const defaultNetworkOptions: NetworkRecordOptions = { type Headers = Record; type NetworkRequest = { - requestUrl: string; - requestMethod: string; - requestInitiator: InitiatorType; + url: string; + method?: string; + initiatorType: InitiatorType; + status?: number; + startTime: number; + endTime: number; requestHeaders?: Headers; requestBody?: string | null; - responseStatus?: number; - responseDuration: number; responseHeaders?: Headers; responseBody?: string | null; }; @@ -127,15 +128,12 @@ function initPerformanceObserver( )), ); cb({ - requests: initialPerformanceEntries.map((performanceEntry) => ({ - requestUrl: performanceEntry.name, - requestMethod: 'GET', - requestInitiator: performanceEntry.initiatorType as InitiatorType, - responseStatus: - 'responseStatus' in performanceEntry - ? performanceEntry.responseStatus - : undefined, - responseDuration: Math.round(performanceEntry.duration), + requests: initialPerformanceEntries.map((entry) => ({ + url: entry.name, + initiatorType: entry.initiatorType as InitiatorType, + status: 'responseStatus' in entry ? entry.responseStatus : undefined, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), })), isInitial: true, }); @@ -154,15 +152,12 @@ function initPerformanceObserver( entry.initiatorType !== 'fetch'), ); cb({ - requests: performanceEntries.map((performanceEntry) => ({ - requestUrl: performanceEntry.name, - requestMethod: 'GET', - requestInitiator: performanceEntry.initiatorType as InitiatorType, - responseStatus: - 'responseStatus' in performanceEntry - ? performanceEntry.responseStatus - : undefined, - responseDuration: Math.round(performanceEntry.duration), + requests: performanceEntries.map((entry) => ({ + url: entry.name, + initiatorType: entry.initiatorType as InitiatorType, + status: 'responseStatus' in entry ? entry.responseStatus : undefined, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), })), }); }); @@ -188,11 +183,11 @@ const getRequestPerformanceEntry = async ( ) as PerformanceResourceTiming[]; const performanceEntry = findLast( urlPerformanceEntries, - (performanceEntry) => - isResourceTiming(performanceEntry) && - performanceEntry.initiatorType === initiatorType && - (!after || performanceEntry.startTime >= after) && - (!before || performanceEntry.startTime <= before), + (entry) => + isResourceTiming(entry) && + entry.initiatorType === initiatorType && + (!after || entry.startTime >= after) && + (!before || entry.startTime <= before), ); if (!performanceEntry) { await new Promise((resolve) => setTimeout(resolve, 50 * attempt)); @@ -322,15 +317,16 @@ function initXhrObserver( after, before, ) - .then((performanceEntry) => { + .then((entry) => { const request: NetworkRequest = { - requestUrl: performanceEntry.name, - requestMethod: req.method, - requestInitiator: performanceEntry.initiatorType as InitiatorType, + url: entry.name, + method: req.method, + initiatorType: entry.initiatorType as InitiatorType, + status: xhr.status, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), requestHeaders: networkRequest.requestHeaders, requestBody: networkRequest.requestBody, - responseStatus: xhr.status, - responseDuration: performanceEntry.duration, responseHeaders: networkRequest.responseHeaders, responseBody: networkRequest.responseBody, }; @@ -409,7 +405,6 @@ function initFetchObserver( after = win.performance.now(); res = await originalFetch(req); before = win.performance.now(); - networkRequest.responseStatus = res.status; if (recordResponseHeaders) { networkRequest.responseHeaders = {}; res.headers.forEach((value, header) => { @@ -437,15 +432,16 @@ function initFetchObserver( return res; } finally { getRequestPerformanceEntry(win, 'fetch', req.url, after, before) - .then((performanceEntry) => { + .then((entry) => { const request: NetworkRequest = { - requestUrl: performanceEntry.name, - requestMethod: req.method, - requestInitiator: performanceEntry.initiatorType as InitiatorType, + url: entry.name, + method: req.method, + initiatorType: entry.initiatorType as InitiatorType, + status: res?.status, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), requestHeaders: networkRequest.requestHeaders, requestBody: networkRequest.requestBody, - responseStatus: res?.status, - responseDuration: performanceEntry.duration, responseHeaders: networkRequest.responseHeaders, responseBody: networkRequest.responseBody, }; From d7db138bb6d3b351f9655fcca29240f82597de9c Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 12:22:10 +0000 Subject: [PATCH 35/43] fix: warning from rollup --- packages/rrweb/src/plugins/network/record/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index bd8260848e..b8a0e209ac 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -167,14 +167,14 @@ function initPerformanceObserver( }; } -const getRequestPerformanceEntry = async ( +async function getRequestPerformanceEntry( win: IWindow, initiatorType: string, url: string, after?: number, before?: number, attempt = 0, -): Promise => { +): Promise { if (attempt > 10) { throw new Error('Cannot find performance entry'); } @@ -201,7 +201,7 @@ const getRequestPerformanceEntry = async ( ); } return performanceEntry; -}; +} function initXhrObserver( cb: networkCallback, From 8fdec479ad09e386834dafa3aec32a384f3258de Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 12:26:54 +0000 Subject: [PATCH 36/43] moved findLast util --- packages/rrweb/src/plugins/network/record/index.ts | 3 ++- packages/rrweb/src/plugins/utils/find-last.ts | 11 +++++++++++ packages/rrweb/src/utils.ts | 12 ------------ 3 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 packages/rrweb/src/plugins/utils/find-last.ts diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index b8a0e209ac..9361e50639 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,6 +1,7 @@ import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; -import { findLast, patch } from '../../../utils'; +import { patch } from '../../../utils'; import { stringify, StringifyOptions } from '../../utils/stringify'; +import { findLast } from '../../utils/find-last'; export type InitiatorType = | 'audio' diff --git a/packages/rrweb/src/plugins/utils/find-last.ts b/packages/rrweb/src/plugins/utils/find-last.ts new file mode 100644 index 0000000000..a614538614 --- /dev/null +++ b/packages/rrweb/src/plugins/utils/find-last.ts @@ -0,0 +1,11 @@ +export function findLast( + array: Array, + predicate: (value: T) => boolean, +): T | undefined { + const length = array.length; + for (let i = length - 1; i >= 0; i -= 1) { + if (predicate(array[i])) { + return array[i]; + } + } +} diff --git a/packages/rrweb/src/utils.ts b/packages/rrweb/src/utils.ts index d61de7c588..1d5ec83855 100644 --- a/packages/rrweb/src/utils.ts +++ b/packages/rrweb/src/utils.ts @@ -545,15 +545,3 @@ export function inDom(n: Node): boolean { if (!doc) return false; return doc.contains(n) || shadowHostInDom(n); } - -export function findLast( - array: Array, - predicate: (value: T) => boolean, -): T | undefined { - const length = array.length; - for (let i = length - 1; i >= 0; i -= 1) { - if (predicate(array[i])) { - return array[i]; - } - } -} From 0f30b295d323aa95896947912afb5e9c801462ba Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 16:41:13 +0000 Subject: [PATCH 37/43] any body --- .../rrweb/src/plugins/network/record/index.ts | 70 +++++-------------- 1 file changed, 16 insertions(+), 54 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 9361e50639..2f8693b6a6 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -1,6 +1,5 @@ import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; import { patch } from '../../../utils'; -import { stringify, StringifyOptions } from '../../utils/stringify'; import { findLast } from '../../utils/find-last'; export type InitiatorType = @@ -29,20 +28,8 @@ export type InitiatorType = type NetworkRecordOptions = { initiatorTypes?: InitiatorType[]; ignoreRequestFn?: (data: NetworkRequest) => boolean; - recordHeaders?: - | boolean - | StringifyOptions - | { - request: boolean | StringifyOptions; - response: boolean | StringifyOptions; - }; - recordBody?: - | boolean - | StringifyOptions - | { - request: boolean | StringifyOptions; - response: boolean | StringifyOptions; - }; + recordHeaders?: boolean | { request: boolean; response: boolean }; + recordBody?: boolean | { request: boolean; response: boolean }; recordInitialRequests?: boolean; }; @@ -86,9 +73,9 @@ type NetworkRequest = { startTime: number; endTime: number; requestHeaders?: Headers; - requestBody?: string | null; + requestBody?: unknown; responseHeaders?: Headers; - responseBody?: string | null; + responseBody?: unknown; }; export type NetworkData = { @@ -265,12 +252,7 @@ function initXhrObserver( if (body === undefined || body === null) { networkRequest.requestBody = null; } else { - networkRequest.requestBody = stringify( - body, - typeof recordRequestBody === 'object' - ? recordRequestBody - : undefined, - ); + networkRequest.requestBody = body; } } after = win.performance.now(); @@ -295,20 +277,10 @@ function initXhrObserver( }); } if (recordResponseBody) { - if (!xhr.response) { + if (xhr.response === undefined || xhr.response === null) { networkRequest.responseBody = null; } else { - try { - const objBody = JSON.parse(xhr.response as string) as object; - networkRequest.responseBody = stringify( - objBody, - typeof recordResponseBody === 'object' - ? recordResponseBody - : undefined, - ); - } catch { - networkRequest.responseBody = xhr.response as string; - } + networkRequest.responseBody = xhr.response; } } getRequestPerformanceEntry( @@ -395,12 +367,7 @@ function initFetchObserver( if (req.body === undefined || req.body === null) { networkRequest.requestBody = null; } else { - networkRequest.requestBody = stringify( - req.body, - typeof recordRequestBody === 'object' - ? recordRequestBody - : undefined, - ); + networkRequest.requestBody = req.body; } } after = win.performance.now(); @@ -413,21 +380,16 @@ function initFetchObserver( }); } if (recordResponseBody) { - const reqBody = await res.clone().text(); - if (!reqBody) { + let body: string | undefined; + try { + body = await res.clone().text(); + } catch { + // + } + if (res.body === undefined || res.body === null) { networkRequest.responseBody = null; } else { - try { - const objBody = JSON.parse(reqBody) as object; - networkRequest.responseBody = stringify( - objBody, - typeof recordResponseBody === 'object' - ? recordResponseBody - : undefined, - ); - } catch { - networkRequest.responseBody = reqBody; - } + networkRequest.responseBody = body; } } return res; From 8c41efd3b015d1dfb55646d71e9aa5a27b1c3bde Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 16:59:07 +0000 Subject: [PATCH 38/43] better patch --- .../rrweb/src/plugins/network/record/index.ts | 139 +++++++++--------- 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 2f8693b6a6..c427336c5c 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -223,7 +223,7 @@ function initXhrObserver( options.recordBody.response); const restorePatch = patch( - XMLHttpRequest.prototype, + win.XMLHttpRequest.prototype, 'open', (originalOpen: typeof XMLHttpRequest.prototype.open) => { return function ( @@ -349,82 +349,79 @@ function initFetchObserver( !('response' in options.recordBody) || options.recordBody.response); - const originalFetch = win.fetch; - const wrappedFetch: typeof fetch = async (url, init) => { - const req = new Request(url, init); - let res: Response | undefined; - const networkRequest: Partial = {}; - let after: number | undefined; - let before: number | undefined; - try { - if (recordRequestHeaders) { - networkRequest.requestHeaders = {}; - req.headers.forEach((value, header) => { - networkRequest.requestHeaders![header] = value; - }); - } - if (recordRequestBody) { - if (req.body === undefined || req.body === null) { - networkRequest.requestBody = null; - } else { - networkRequest.requestBody = req.body; + const restorePatch = patch(win, 'fetch', (originalFetch: typeof fetch) => { + return async function ( + url: URL | RequestInfo, + init?: RequestInit | undefined, + ) { + const req = new Request(url, init); + let res: Response | undefined; + const networkRequest: Partial = {}; + let after: number | undefined; + let before: number | undefined; + try { + if (recordRequestHeaders) { + networkRequest.requestHeaders = {}; + req.headers.forEach((value, header) => { + networkRequest.requestHeaders![header] = value; + }); } - } - after = win.performance.now(); - res = await originalFetch(req); - before = win.performance.now(); - if (recordResponseHeaders) { - networkRequest.responseHeaders = {}; - res.headers.forEach((value, header) => { - networkRequest.responseHeaders![header] = value; - }); - } - if (recordResponseBody) { - let body: string | undefined; - try { - body = await res.clone().text(); - } catch { - // + if (recordRequestBody) { + if (req.body === undefined || req.body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = req.body; + } + } + after = win.performance.now(); + res = await originalFetch(req); + before = win.performance.now(); + if (recordResponseHeaders) { + networkRequest.responseHeaders = {}; + res.headers.forEach((value, header) => { + networkRequest.responseHeaders![header] = value; + }); } - if (res.body === undefined || res.body === null) { - networkRequest.responseBody = null; - } else { - networkRequest.responseBody = body; + if (recordResponseBody) { + let body: string | undefined; + try { + body = await res.clone().text(); + } catch { + // + } + if (res.body === undefined || res.body === null) { + networkRequest.responseBody = null; + } else { + networkRequest.responseBody = body; + } } + return res; + } finally { + getRequestPerformanceEntry(win, 'fetch', req.url, after, before) + .then((entry) => { + const request: NetworkRequest = { + url: entry.name, + method: req.method, + initiatorType: entry.initiatorType as InitiatorType, + status: res?.status, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), + requestHeaders: networkRequest.requestHeaders, + requestBody: networkRequest.requestBody, + responseHeaders: networkRequest.responseHeaders, + responseBody: networkRequest.responseBody, + }; + cb({ requests: [request] }); + }) + .catch(() => { + // + }); } - return res; - } finally { - getRequestPerformanceEntry(win, 'fetch', req.url, after, before) - .then((entry) => { - const request: NetworkRequest = { - url: entry.name, - method: req.method, - initiatorType: entry.initiatorType as InitiatorType, - status: res?.status, - startTime: Math.round(entry.startTime), - endTime: Math.round(entry.responseEnd), - requestHeaders: networkRequest.requestHeaders, - requestBody: networkRequest.requestBody, - responseHeaders: networkRequest.responseHeaders, - responseBody: networkRequest.responseBody, - }; - cb({ requests: [request] }); - }) - .catch(() => { - // - }); - } - }; - wrappedFetch.prototype = {}; - Object.defineProperties(wrappedFetch, { - __rrweb_original__: { - enumerable: false, - value: originalFetch, - }, + }; }); - win.fetch = wrappedFetch; + return () => { - win.fetch = originalFetch; + restorePatch(); }; } From cae71f76a2b845d8215e0fef8d787973ca1e27cb Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 17:41:04 +0000 Subject: [PATCH 39/43] content-type body filtering --- .../rrweb/src/plugins/network/record/index.ts | 135 +++++++++++------- 1 file changed, 86 insertions(+), 49 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index c427336c5c..5bcd5dead9 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -29,7 +29,10 @@ type NetworkRecordOptions = { initiatorTypes?: InitiatorType[]; ignoreRequestFn?: (data: NetworkRequest) => boolean; recordHeaders?: boolean | { request: boolean; response: boolean }; - recordBody?: boolean | { request: boolean; response: boolean }; + recordBody?: + | boolean + | string[] + | { request: boolean | string[]; response: boolean | string[] }; recordInitialRequests?: boolean; }; @@ -155,6 +158,13 @@ function initPerformanceObserver( }; } +function getContentType(headers: Headers) { + const contentTypeHeader = Object.keys(headers).find( + (key) => key.toLowerCase() === 'content-type', + ); + return contentTypeHeader && headers[contentTypeHeader]; +} + async function getRequestPerformanceEntry( win: IWindow, initiatorType: string, @@ -238,21 +248,28 @@ function initXhrObserver( const networkRequest: Partial = {}; let after: number | undefined; let before: number | undefined; + const requestHeaders: Headers = {}; + const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); + xhr.setRequestHeader = (header: string, value: string) => { + requestHeaders[header] = value; + return originalSetRequestHeader(header, value); + }; if (recordRequestHeaders) { - networkRequest.requestHeaders = {}; - const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); - xhr.setRequestHeader = (header: string, value: string) => { - networkRequest.requestHeaders![header] = value; - return originalSetRequestHeader(header, value); - }; + networkRequest.requestHeaders = requestHeaders; } const originalSend = xhr.send.bind(xhr); xhr.send = (body) => { if (recordRequestBody) { - if (body === undefined || body === null) { - networkRequest.requestBody = null; - } else { - networkRequest.requestBody = body; + const contentType = getContentType(requestHeaders); + if ( + recordRequestBody === true || + (contentType && recordRequestBody.includes(contentType)) + ) { + if (body === undefined || body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = body; + } } } after = win.performance.now(); @@ -263,24 +280,31 @@ function initXhrObserver( return; } before = win.performance.now(); + const responseHeaders: Headers = {}; + const rawHeaders = xhr.getAllResponseHeaders(); + const headers = rawHeaders.trim().split(/[\r\n]+/); + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (header) { + responseHeaders[header] = value; + } + }); if (recordResponseHeaders) { - networkRequest.responseHeaders = {}; - const rawHeaders = xhr.getAllResponseHeaders(); - const headers = rawHeaders.trim().split(/[\r\n]+/); - headers.forEach((line) => { - const parts = line.split(': '); - const header = parts.shift(); - const value = parts.join(': '); - if (header) { - networkRequest.responseHeaders![header] = value; - } - }); + networkRequest.responseHeaders = responseHeaders; } if (recordResponseBody) { - if (xhr.response === undefined || xhr.response === null) { - networkRequest.responseBody = null; - } else { - networkRequest.responseBody = xhr.response; + const contentType = getContentType(responseHeaders); + if ( + recordResponseBody === true || + (contentType && recordResponseBody.includes(contentType)) + ) { + if (xhr.response === undefined || xhr.response === null) { + networkRequest.responseBody = null; + } else { + networkRequest.responseBody = xhr.response; + } } } getRequestPerformanceEntry( @@ -360,39 +384,53 @@ function initFetchObserver( let after: number | undefined; let before: number | undefined; try { + const requestHeaders: Headers = {}; + req.headers.forEach((value, header) => { + requestHeaders[header] = value; + }); if (recordRequestHeaders) { - networkRequest.requestHeaders = {}; - req.headers.forEach((value, header) => { - networkRequest.requestHeaders![header] = value; - }); + networkRequest.requestHeaders = requestHeaders; } if (recordRequestBody) { - if (req.body === undefined || req.body === null) { - networkRequest.requestBody = null; - } else { - networkRequest.requestBody = req.body; + const contentType = getContentType(requestHeaders); + if ( + recordRequestBody === true || + (contentType && recordRequestBody.includes(contentType)) + ) { + if (req.body === undefined || req.body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = req.body; + } } } after = win.performance.now(); res = await originalFetch(req); before = win.performance.now(); + const responseHeaders: Headers = {}; + res.headers.forEach((value, header) => { + responseHeaders[header] = value; + }); if (recordResponseHeaders) { - networkRequest.responseHeaders = {}; - res.headers.forEach((value, header) => { - networkRequest.responseHeaders![header] = value; - }); + networkRequest.responseHeaders = responseHeaders; } if (recordResponseBody) { - let body: string | undefined; - try { - body = await res.clone().text(); - } catch { - // - } - if (res.body === undefined || res.body === null) { - networkRequest.responseBody = null; - } else { - networkRequest.responseBody = body; + const contentType = getContentType(responseHeaders); + if ( + recordResponseBody === true || + (contentType && recordResponseBody.includes(contentType)) + ) { + let body: string | undefined; + try { + body = await res.clone().text(); + } catch { + // + } + if (res.body === undefined || res.body === null) { + networkRequest.responseBody = null; + } else { + networkRequest.responseBody = body; + } } } return res; @@ -419,7 +457,6 @@ function initFetchObserver( } }; }); - return () => { restorePatch(); }; From d86043316e169413979fce132aa773683233373b Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 7 Feb 2023 17:44:44 +0000 Subject: [PATCH 40/43] update docs --- docs/recipes/network.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/recipes/network.md b/docs/recipes/network.md index 5764872bcd..b7a4f5f567 100644 --- a/docs/recipes/network.md +++ b/docs/recipes/network.md @@ -37,8 +37,8 @@ rrweb.record({ rrweb.getRecordConsolePlugin({ initiatorTypes: ['fetch', 'xmlhttprequest'], // block recording event for request to upload events to server - ignoreRequestFn: ({ performanceEntry }) => { - if (performanceEntry.name === 'https://api.my-server.com/events') { + ignoreRequestFn: (request) => { + if (request.url === 'https://api.my-server.com/events') { return true; } return false; @@ -72,9 +72,9 @@ const replayer = new rrweb.Replayer(events, { rrweb.getReplayNetworkPlugin({ onNetworkData: ({ requests }) => { for (const request of requests) { - const url = request.requestUrl; - const method = request.responseMethod; - const status = request.responseStatus; + const url = request.url; + const method = request.method; + const status = request.status; console.log(`${method} ${url} ${status}`); } }, From 027811306b697b28a611004af63e113a10dc5bc6 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 8 Feb 2023 09:56:27 +0000 Subject: [PATCH 41/43] better body typing --- .../rrweb/src/plugins/network/record/index.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index 5bcd5dead9..fab283f1d3 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -67,6 +67,16 @@ const defaultNetworkOptions: NetworkRecordOptions = { }; type Headers = Record; +type Body = + | string + | Document + | Blob + | ArrayBufferView + | ArrayBuffer + | FormData + | URLSearchParams + | ReadableStream + | null; type NetworkRequest = { url: string; @@ -76,9 +86,9 @@ type NetworkRequest = { startTime: number; endTime: number; requestHeaders?: Headers; - requestBody?: unknown; + requestBody?: Body; responseHeaders?: Headers; - responseBody?: unknown; + responseBody?: Body; }; export type NetworkData = { @@ -303,6 +313,7 @@ function initXhrObserver( if (xhr.response === undefined || xhr.response === null) { networkRequest.responseBody = null; } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment networkRequest.responseBody = xhr.response; } } From 6804e4be140b04258c6d99ab58721f4e0989bf5b Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 8 Feb 2023 14:13:31 +0000 Subject: [PATCH 42/43] better recordBody --- .../rrweb/src/plugins/network/record/index.ts | 169 ++++++++---------- 1 file changed, 72 insertions(+), 97 deletions(-) diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts index fab283f1d3..2c59e78844 100644 --- a/packages/rrweb/src/plugins/network/record/index.ts +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -168,11 +168,34 @@ function initPerformanceObserver( }; } -function getContentType(headers: Headers) { - const contentTypeHeader = Object.keys(headers).find( - (key) => key.toLowerCase() === 'content-type', +function shouldRecordHeaders( + type: 'request' | 'response', + recordHeaders: NetworkRecordOptions['recordHeaders'], +) { + return ( + !!recordHeaders && + (typeof recordHeaders === 'boolean' || recordHeaders[type]) ); - return contentTypeHeader && headers[contentTypeHeader]; +} + +function shouldRecordBody( + type: 'request' | 'response', + recordBody: NetworkRecordOptions['recordBody'], + headers: Headers, +) { + function matchesContentType(contentTypes: string[]) { + const contentTypeHeader = Object.keys(headers).find( + (key) => key.toLowerCase() === 'content-type', + ); + const contentType = contentTypeHeader && headers[contentTypeHeader]; + return contentTypes.some((ct) => contentType?.includes(ct)); + } + if (!recordBody) return false; + if (typeof recordBody === 'boolean') return true; + if (Array.isArray(recordBody)) return matchesContentType(recordBody); + const recordBodyType = recordBody[type]; + if (typeof recordBodyType === 'boolean') return recordBodyType; + return matchesContentType(recordBodyType); } async function getRequestPerformanceEntry( @@ -221,27 +244,14 @@ function initXhrObserver( // }; } - const recordRequestHeaders = - !!options.recordHeaders && - (typeof options.recordHeaders === 'boolean' || - !('request' in options.recordHeaders) || - options.recordHeaders.request); - const recordRequestBody = - !!options.recordBody && - (typeof options.recordBody === 'boolean' || - !('request' in options.recordBody) || - options.recordBody.request); - const recordResponseHeaders = - !!options.recordHeaders && - (typeof options.recordHeaders === 'boolean' || - !('response' in options.recordHeaders) || - options.recordHeaders.response); - const recordResponseBody = - !!options.recordBody && - (typeof options.recordBody === 'boolean' || - !('response' in options.recordBody) || - options.recordBody.response); - + const recordRequestHeaders = shouldRecordHeaders( + 'request', + options.recordHeaders, + ); + const recordResponseHeaders = shouldRecordHeaders( + 'response', + options.recordHeaders, + ); const restorePatch = patch( win.XMLHttpRequest.prototype, 'open', @@ -269,17 +279,11 @@ function initXhrObserver( } const originalSend = xhr.send.bind(xhr); xhr.send = (body) => { - if (recordRequestBody) { - const contentType = getContentType(requestHeaders); - if ( - recordRequestBody === true || - (contentType && recordRequestBody.includes(contentType)) - ) { - if (body === undefined || body === null) { - networkRequest.requestBody = null; - } else { - networkRequest.requestBody = body; - } + if (shouldRecordBody('request', options.recordBody, requestHeaders)) { + if (body === undefined || body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = body; } } after = win.performance.now(); @@ -304,18 +308,14 @@ function initXhrObserver( if (recordResponseHeaders) { networkRequest.responseHeaders = responseHeaders; } - if (recordResponseBody) { - const contentType = getContentType(responseHeaders); - if ( - recordResponseBody === true || - (contentType && recordResponseBody.includes(contentType)) - ) { - if (xhr.response === undefined || xhr.response === null) { - networkRequest.responseBody = null; - } else { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - networkRequest.responseBody = xhr.response; - } + if ( + shouldRecordBody('response', options.recordBody, responseHeaders) + ) { + if (xhr.response === undefined || xhr.response === null) { + networkRequest.responseBody = null; + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + networkRequest.responseBody = xhr.response; } } getRequestPerformanceEntry( @@ -363,27 +363,14 @@ function initFetchObserver( // }; } - const recordRequestHeaders = - !!options.recordHeaders && - (typeof options.recordHeaders === 'boolean' || - !('request' in options.recordHeaders) || - options.recordHeaders.request); - const recordRequestBody = - !!options.recordBody && - (typeof options.recordBody === 'boolean' || - !('request' in options.recordBody) || - options.recordBody.request); - const recordResponseHeaders = - !!options.recordHeaders && - (typeof options.recordHeaders === 'boolean' || - !('response' in options.recordHeaders) || - options.recordHeaders.response); - const recordResponseBody = - !!options.recordBody && - (typeof options.recordBody === 'boolean' || - !('response' in options.recordBody) || - options.recordBody.response); - + const recordRequestHeaders = shouldRecordHeaders( + 'request', + options.recordHeaders, + ); + const recordResponseHeaders = shouldRecordHeaders( + 'response', + options.recordHeaders, + ); const restorePatch = patch(win, 'fetch', (originalFetch: typeof fetch) => { return async function ( url: URL | RequestInfo, @@ -402,17 +389,11 @@ function initFetchObserver( if (recordRequestHeaders) { networkRequest.requestHeaders = requestHeaders; } - if (recordRequestBody) { - const contentType = getContentType(requestHeaders); - if ( - recordRequestBody === true || - (contentType && recordRequestBody.includes(contentType)) - ) { - if (req.body === undefined || req.body === null) { - networkRequest.requestBody = null; - } else { - networkRequest.requestBody = req.body; - } + if (shouldRecordBody('request', options.recordBody, requestHeaders)) { + if (req.body === undefined || req.body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = req.body; } } after = win.performance.now(); @@ -425,23 +406,17 @@ function initFetchObserver( if (recordResponseHeaders) { networkRequest.responseHeaders = responseHeaders; } - if (recordResponseBody) { - const contentType = getContentType(responseHeaders); - if ( - recordResponseBody === true || - (contentType && recordResponseBody.includes(contentType)) - ) { - let body: string | undefined; - try { - body = await res.clone().text(); - } catch { - // - } - if (res.body === undefined || res.body === null) { - networkRequest.responseBody = null; - } else { - networkRequest.responseBody = body; - } + if (shouldRecordBody('response', options.recordBody, responseHeaders)) { + let body: string | undefined; + try { + body = await res.clone().text(); + } catch { + // + } + if (res.body === undefined || res.body === null) { + networkRequest.responseBody = null; + } else { + networkRequest.responseBody = body; } } return res; From 8863576a1abb51366c0156d8d8b64c9656baf0ee Mon Sep 17 00:00:00 2001 From: James Berry <69924001+jlalmes@users.noreply.github.com> Date: Mon, 20 Feb 2023 10:19:46 +0000 Subject: [PATCH 43/43] Update docs/recipes/network.md Co-authored-by: Ben White --- docs/recipes/network.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recipes/network.md b/docs/recipes/network.md index b7a4f5f567..b62b77a939 100644 --- a/docs/recipes/network.md +++ b/docs/recipes/network.md @@ -34,7 +34,7 @@ rrweb.record({ }, // customized record options plugins: [ - rrweb.getRecordConsolePlugin({ + rrweb.getRecordNetworkPlugin({ initiatorTypes: ['fetch', 'xmlhttprequest'], // block recording event for request to upload events to server ignoreRequestFn: (request) => {