From 02600b033efd8ee53a1e496ee4e25fbc4bc455ff Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Fri, 6 Oct 2023 14:40:48 -0700 Subject: [PATCH 1/3] [js/webgpu] allow to specify callback for profiling data --- js/common/lib/env.ts | 37 ++++++++++++++++ js/web/lib/wasm/jsep/init.ts | 6 ++- .../lib/wasm/jsep/webgpu/program-manager.ts | 43 +++++++++++++------ js/web/test/test-main.ts | 2 +- 4 files changed, 73 insertions(+), 15 deletions(-) diff --git a/js/common/lib/env.ts b/js/common/lib/env.ts index 76575ef7b9368..0cded7e5edbcb 100644 --- a/js/common/lib/env.ts +++ b/js/common/lib/env.ts @@ -92,11 +92,48 @@ export declare namespace Env { async?: boolean; } + export interface WebGpuProfilingDataV1TensorMetadata { + dims: readonly number[]; + dataType: string; + } + export interface WebGpuProfilingDataV1 { + version: 1; + inputsMetadata: readonly WebGpuProfilingDataV1TensorMetadata[]; + outputsMetadata: readonly WebGpuProfilingDataV1TensorMetadata[]; + kernelId: number; + kernelType: string; + kernelName: string; + startTime: number; + endTime: number; + } + + export type WebGpuProfilingData = WebGpuProfilingDataV1; + export interface WebGpuFlags { /** * Set or get the profiling mode. + * + * @deprecated Use `env.webgpu.profiling.mode` instead. If `env.webgpu.profiling.mode` is set, this property will be + * ignored. */ profilingMode?: 'off'|'default'; + /** + * Set or get the profiling configuration. + */ + profiling?: { + /** + * Set or get the profiling mode. + * + * @defaultValue `'off'` + */ + mode?: 'off'|'default'; + + /** + * Set or get a callback function when a profiling data is received. If not set, the profiling data will be + * printed to console. + */ + ondata?: (data: WebGpuProfilingData) => void; + }; /** * Get the device for WebGPU. * diff --git a/js/web/lib/wasm/jsep/init.ts b/js/web/lib/wasm/jsep/init.ts index 6ff3971d720fd..4fd72225f94ef 100644 --- a/js/web/lib/wasm/jsep/init.ts +++ b/js/web/lib/wasm/jsep/init.ts @@ -176,8 +176,10 @@ export const init = async(module: OrtWasmModule, env: Env): Promise => { // jsepCreateKernel (name: string, kernel: number, attribute: unknown) => backend.createKernel( name, kernel, attribute, - env.debug || env.webgpu.profilingMode === 'default' ? module.UTF8ToString(module._JsepGetNodeName(kernel)) : - `${kernel}`), + env.debug || env.webgpu.profiling?.mode === 'default' || + (!env.webgpu.profiling?.mode && env.webgpu.profilingMode === 'default') ? + module.UTF8ToString(module._JsepGetNodeName(kernel)) : + `${kernel}`), // jsepReleaseKernel (kernel: number) => backend.releaseKernel(kernel), diff --git a/js/web/lib/wasm/jsep/webgpu/program-manager.ts b/js/web/lib/wasm/jsep/webgpu/program-manager.ts index cf2687e4c7382..3101dd2182bac 100644 --- a/js/web/lib/wasm/jsep/webgpu/program-manager.ts +++ b/js/web/lib/wasm/jsep/webgpu/program-manager.ts @@ -36,7 +36,10 @@ export class ProgramManager { dispatchGroup: [number, number, number]): void { const device = this.backend.device; const computePassEncoder = this.backend.getComputePassEncoder(); - const profilingEnabled = this.backend.supportTimestampQuery && this.backend.env.webgpu.profilingMode === 'default'; + const webgpuEnv = this.backend.env.webgpu; + const profilingEnabled = this.backend.supportTimestampQuery && + (webgpuEnv.profiling?.mode === 'default' || + (!webgpuEnv.profiling?.mode && webgpuEnv.profilingMode === 'default')); if (profilingEnabled) { // profiling write start timestamp @@ -103,17 +106,33 @@ export class ProgramManager { } this.backend.gpuDataManager.release(syncData.id); - let inputShapes = ''; - inputsTensorView.forEach((value, i) => { - inputShapes += `input[${i}]: [${value.dims}] | ${tensorDataTypeEnumToString(value.dataType)}, `; - }); - let outputShapes = ''; - buildArtifact.programInfo.outputs.forEach((value, i) => { - outputShapes += `output[${i}]: [${value.dims}] | ${tensorDataTypeEnumToString(value.dataType)}, `; - }); - // eslint-disable-next-line no-console - console.log(`[profiling] kernel "${kernelId}|${kernelName}" ${inputShapes}${outputShapes}execution time: ${ - endTime - startTime} ns`); + if (this.backend.env.webgpu.profiling?.ondata) { + this.backend.env.webgpu.profiling.ondata({ + version: 1, + inputsMetadata: inputsTensorView.map( + value => ({dims: value.dims, dataType: tensorDataTypeEnumToString(value.dataType)})), + outputsMetadata: buildArtifact.programInfo.outputs.map( + value => ({dims: value.dims, dataType: tensorDataTypeEnumToString(value.dataType)})), + kernelId, + kernelType: kernelInfo[0], + kernelName: kernelInfo[1], + startTime, + endTime, + }); + } else { + // if no callback is provided, print the profiling message to console + let inputShapes = ''; + inputsTensorView.forEach((value, i) => { + inputShapes += `input[${i}]: [${value.dims}] | ${tensorDataTypeEnumToString(value.dataType)}, `; + }); + let outputShapes = ''; + buildArtifact.programInfo.outputs.forEach((value, i) => { + outputShapes += `output[${i}]: [${value.dims}] | ${tensorDataTypeEnumToString(value.dataType)}, `; + }); + // eslint-disable-next-line no-console + console.log(`[profiling] kernel "${kernelId}|${kernelName}" ${inputShapes}${outputShapes}execution time: ${ + endTime - startTime} ns`); + } }); } diff --git a/js/web/test/test-main.ts b/js/web/test/test-main.ts index 24ab0694b32b8..9bd0ec1425f95 100644 --- a/js/web/test/test-main.ts +++ b/js/web/test/test-main.ts @@ -56,7 +56,7 @@ if (options.globalEnvFlags) { ort.env.wasm.initTimeout = flags.wasm.initTimeout; } if (flags.webgpu?.profilingMode !== undefined) { - ort.env.webgpu.profilingMode = flags.webgpu.profilingMode; + ort.env.webgpu.profiling = {mode: flags.webgpu.profilingMode}; } if (flags.webgpu?.validateInputContent !== undefined) { ort.env.webgpu.validateInputContent = flags.webgpu.validateInputContent; From 08c0f232ff30188f65beec429948805a4b636f3e Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 11 Oct 2023 17:29:57 -0700 Subject: [PATCH 2/3] revise program-manager.ts --- js/web/lib/wasm/jsep/webgpu/program-manager.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/js/web/lib/wasm/jsep/webgpu/program-manager.ts b/js/web/lib/wasm/jsep/webgpu/program-manager.ts index 33c67159b9988..75f4ae3ab6100 100644 --- a/js/web/lib/wasm/jsep/webgpu/program-manager.ts +++ b/js/web/lib/wasm/jsep/webgpu/program-manager.ts @@ -90,12 +90,11 @@ export class ProgramManager { const kernelId = this.backend.currentKernelId!; const kernelInfo = this.backend.kernels.get(kernelId)!; - const kernelName = `[${kernelInfo[0]}] ${kernelInfo[1]}`; syncData.buffer.mapAsync(GPUMapMode.READ).then(() => { const mappedData = new BigUint64Array(syncData.buffer.getMappedRange()); - const startTimeU64 = mappedData[0]; - const endTimeU64 = mappedData[1]; + const [startTimeU64, endTimeU64] = mappedData; + const [kernelType, kernelName] = kernelInfo; syncData.buffer.unmap(); @@ -119,8 +118,8 @@ export class ProgramManager { outputsMetadata: outputTensorViews.map( value => ({dims: value.dims, dataType: tensorDataTypeEnumToString(value.dataType)})), kernelId, - kernelType: kernelInfo[0], - kernelName: kernelInfo[1], + kernelType, + kernelName, startTime, endTime, }); @@ -135,8 +134,8 @@ export class ProgramManager { outputShapes += `output[${i}]: [${value.dims}] | ${tensorDataTypeEnumToString(value.dataType)}, `; }); // eslint-disable-next-line no-console - console.log(`[profiling] kernel "${kernelId}|${kernelName}" ${inputShapes}${outputShapes}execution time: ${ - endTime - startTime} ns`); + console.log(`[profiling] kernel "${kernelId}|[${kernelType}] ${kernelName}" ${inputShapes}${ + outputShapes}execution time: ${endTime - startTime} ns`); } }); } From 4d4f664303007769ba43f361bf80980416cffc5d Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Tue, 5 Dec 2023 18:13:02 -0800 Subject: [PATCH 3/3] update to latest --- js/web/lib/wasm/jsep/backend-webgpu.ts | 8 +++----- js/web/lib/wasm/jsep/init.ts | 5 +---- js/web/lib/wasm/jsep/webgpu/program-manager.ts | 13 +------------ 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/js/web/lib/wasm/jsep/backend-webgpu.ts b/js/web/lib/wasm/jsep/backend-webgpu.ts index bb86f147c9c7e..4f4a06c37a94f 100644 --- a/js/web/lib/wasm/jsep/backend-webgpu.ts +++ b/js/web/lib/wasm/jsep/backend-webgpu.ts @@ -254,11 +254,9 @@ export class WebGpuBackend { } isQueryEnabled(): boolean { - if (this.device.features.has('timestamp-query') && this.env.webgpu.profilingMode === 'default') { - return true; - } else { - return false; - } + return this.device.features.has('timestamp-query') && + (this.env.webgpu.profiling?.mode === 'default' || + (!this.env.webgpu.profiling?.mode && this.env.webgpu.profilingMode === 'default')); } /** diff --git a/js/web/lib/wasm/jsep/init.ts b/js/web/lib/wasm/jsep/init.ts index 8c953cfa2236e..e6db631c44eea 100644 --- a/js/web/lib/wasm/jsep/init.ts +++ b/js/web/lib/wasm/jsep/init.ts @@ -175,10 +175,7 @@ export const init = async(module: OrtWasmModule, env: Env): Promise => { // jsepCreateKernel (name: string, kernel: number, attribute: unknown) => backend.createKernel( name, kernel, attribute, - env.debug || env.webgpu.profiling?.mode === 'default' || - (!env.webgpu.profiling?.mode && env.webgpu.profilingMode === 'default') ? - module.UTF8ToString(module._JsepGetNodeName(kernel)) : - `${kernel}`), + env.debug || backend.isQueryEnabled() ? module.UTF8ToString(module._JsepGetNodeName(kernel)) : `${kernel}`), // jsepReleaseKernel (kernel: number) => backend.releaseKernel(kernel), diff --git a/js/web/lib/wasm/jsep/webgpu/program-manager.ts b/js/web/lib/wasm/jsep/webgpu/program-manager.ts index 8c411b934e57b..adf0b1b2964b5 100644 --- a/js/web/lib/wasm/jsep/webgpu/program-manager.ts +++ b/js/web/lib/wasm/jsep/webgpu/program-manager.ts @@ -38,17 +38,6 @@ export class ProgramManager { const device = this.backend.device; const computePassEncoder = this.backend.getComputePassEncoder(); - const webgpuEnv = this.backend.env.webgpu; - const profilingEnabled = this.backend.supportTimestampQuery && - (webgpuEnv.profiling?.mode === 'default' || - (!webgpuEnv.profiling?.mode && webgpuEnv.profilingMode === 'default')); - if (profilingEnabled) { - // profiling write start timestamp - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (computePassEncoder as any).writeTimestamp(this.backend.profilingQuerySet, 0); - } - computePassEncoder.setPipeline(buildArtifact.computePipeline); const entries = []; for (const input of inputs) { @@ -130,7 +119,7 @@ export class ProgramManager { outputShapes += `output[${i}]: [${value.dims}] | ${tensorDataTypeEnumToString(value.dataType)}, `; }); // eslint-disable-next-line no-console - console.log(`[profiling] kernel "${kernelId}|[${kernelType}] ${kernelName}" ${inputShapes}${ + console.log(`[profiling] kernel "${kernelId}|${kernelName}|${buildArtifact.programInfo.name}" ${inputShapes}${ outputShapes}execution time: ${endTime - startTime} ns`); } });