diff --git a/src/lib/assets/js/ort_utils.js b/src/lib/assets/js/ort_utils.js index 4e26397..782317c 100644 --- a/src/lib/assets/js/ort_utils.js +++ b/src/lib/assets/js/ort_utils.js @@ -1,4 +1,41 @@ -import ort from 'onnxruntime-web'; +import * as ort from 'onnxruntime-web'; +import { models } from '../../config'; +import { updateTestQueueStatus, addResult, updateInfo, sleep, catchEm, median } from '../js/utils'; +import { testQueueStore, testQueueLengthStore, resultsStore, numberOfRunsStore } from '../../store/store' + +/** + * @type {number} + */ +export let numOfRuns; + +numberOfRunsStore.subscribe((value) => { + numOfRuns = value; +}); + +/** + * @type {string[]} + */ +export let testQueue; +testQueueStore.subscribe((value) => { + testQueue = value; +}); + +/** + * @type {number} + */ +export let testQueueLength; + +testQueueLengthStore.subscribe((value) => { + testQueueLength = value; +}); + +/** + * @type {string[]} + */ +export let results; +resultsStore.subscribe((value) => { + results = value; +}); export const generateTensor = (dataType, shape, val) => { let size = 1; @@ -61,3 +98,203 @@ export const clone = (x) => { } return feed; } + +const l = (i) => { + console.log(i); +} + +const getUrlById = (id) => { + for (let i = 0; i < models.length; i++) { + if (models[i].id === id) { + return models[i].url; + } + } + return null; +} + +const getInputShapeById = (id) => { + for (let i = 0; i < models.length; i++) { + if (models[i].id === id) { + return models[i].inputshape; + } + } + return null; +} + +const getFeeds = (_model, _dataType) => { + let feed = {}; + let datatype = 'float32'; + switch (_dataType) { + case 'fp32': + datatype = 'float32'; + break; + case 'fp16': + datatype = 'float16'; + break; + case 'int64': + datatype = 'int64'; + break; + case 'uint64': + datatype = 'uint64'; + break; + case 'int32': + datatype = 'int32'; + break; + case 'uint32': + datatype = 'uint32'; + break; + case 'int8': + datatype = 'int8'; + break; + case 'uint8': + datatype = 'uint8'; + break; + default: + datatype = 'float32'; + break; + } + + switch (_model) { + case 'mobilenet_v2': + feed["input"] = generateTensor(datatype, getInputShapeById(_model), 0.5); + break; + default: + break; + } + + if (feed) { + // Without clone(), you'll get DOMException: Failed to execute 'postMessage' on 'Worker': ArrayBuffer at index 0 is already detached. + return feed; + } + +} + +const main = async (_id, _model, _modelType, _dataType, _backend) => { + + let backend = 'wasm'; + let wasmSimd = true; + let numThreads = 1; + let deviceType = 'cpu'; + + switch (_backend) { + case 'wasm_1': + backend = 'wasm'; + wasmSimd = true; + numThreads = 1; + deviceType = 'cpu'; + break; + case 'wasm_4': + backend = 'wasm'; + wasmSimd = true; + numThreads = 4; + deviceType = 'cpu'; + break; + case 'webgl': + backend = 'webgl'; + wasmSimd = false; + numThreads = 1; + deviceType = 'gpu'; + break; + case 'webgpu': + backend = 'webgpu'; + wasmSimd = false; + numThreads = 1; + deviceType = 'gpu'; + break; + case 'webnn_cpu_1': + backend = 'webnn'; + wasmSimd = true; + numThreads = 1; + deviceType = 'cpu'; + break; + case 'webnn_cpu_4': + backend = 'webnn'; + wasmSimd = true; + numThreads = 4; + deviceType = 'cpu'; + break; + case 'webnn_gpu': + backend = 'webnn'; + wasmSimd = true; + numThreads = 1; + deviceType = 'gpu'; + break; + case 'webnn_npu': + backend = 'webnn'; + wasmSimd = true; + numThreads = 1; + deviceType = 'npu'; + break; + default: + backend = 'wasm'; + wasmSimd = true; + numThreads = 1; + deviceType = 'cpu'; + break; + } + + if (_modelType === 'onnx') { + // https://github.com/microsoft/onnxruntime/blob/main/js/common/lib/env.ts + ort.env.wasm.numThreads = numThreads; + ort.env.wasm.simd = wasmSimd; + // ort.env.wasm.wasmPaths = '../node_modules/onnxruntime-web/dist/' + ort.env.wasm.wasmPaths = 'https://10.239.115.52:5173/node_modules/onnxruntime-web/dist/' + ort.env.wasm.proxy = true; + // ort.env.logLevel = "verbose"; // "error"; + // ort.env.debug = false; + + updateTestQueueStatus(_id, 2); + addResult(_model, _modelType, _dataType, _backend, 1, []); + + let modelPath = getUrlById(_model); + + const options = { + executionProviders: [ + { + name: backend, + deviceType: deviceType, + powerPreference: "default", + }, + ], + //executionProviders: [{name: "webnn", deviceType: 'gpu', powerPreference: 'high-performance' }], + }; + + addResult(_model, _modelType, _dataType, _backend, 2, 0, [], 0); + updateInfo(`${testQueueLength - testQueue.length}/${testQueueLength} Testing ${_model} (${_modelType}/${_dataType}) with ${backend} backend ...`); + updateInfo(`${testQueueLength - testQueue.length}/${testQueueLength} Creating session for ${_model} (${_modelType}/${_dataType}) with ${backend} backend ...`); + + const sess = await ort.InferenceSession.create(modelPath, options); + updateInfo(`${testQueueLength - testQueue.length}/${testQueueLength} Warming up ${_model} (${_modelType}/${_dataType}) with ${backend} backend ...`); + let feeds = clone(getFeeds(_model, _dataType)); + + let warmupTime = 0; + const warmupstart = performance.now(); + await sess.run(feeds); + warmupTime = performance.now() - warmupstart; + + let inferenceTimes = []; + for (var i = 0; i < numOfRuns; i++) { + const start = performance.now(); + feeds = clone(getFeeds(_model, _dataType)); + const outputs = await sess.run(feeds); + inferenceTimes.push(performance.now() - start); + } + + let inferenceTimesMedian = parseFloat(median(inferenceTimes).toFixed(2)); + + addResult(_model, _modelType, _dataType, _backend, 3, warmupTime, inferenceTimes, inferenceTimesMedian); + updateInfo(`${testQueueLength - testQueue.length}/${testQueueLength} Test ${_model} (${_modelType}/${_dataType}) with ${backend} backend completed`); + } +} + +export const catchMain = async (_id, _model, _modelType, _dataType, _backend) => { + // const [err, data] = await catchEm(main(_id, _model, _modelType, _dataType, _backend)); + // if (err) { + // addResult(_model, _modelType, _dataType, _backend, 4, []); + // updateInfo(`${testQueueLength - testQueue.length}/${testQueueLength} Error: ${_model} (${_modelType}/${_dataType}) with ${_backend} backend ...`); + // } else { + // // use data + // } + + await main(_id, _model, _modelType, _dataType, _backend); +} \ No newline at end of file diff --git a/src/lib/assets/js/utils.js b/src/lib/assets/js/utils.js index 46748ac..adfc274 100644 --- a/src/lib/assets/js/utils.js +++ b/src/lib/assets/js/utils.js @@ -1,5 +1,6 @@ import { autoStore, infoStore, numberOfRunsStore, backendsStore, dataTypesStore, modelTypesStore, modelsStore, testQueueStore, testQueueLengthStore, resultsStore } from '../../store/store' import { models, uniqueBackends } from '../../config'; +import { catchMain } from '../js/ort_utils' import { goto } from '$app/navigation'; import { base } from '$app/paths'; import { environment } from '$lib/config.js'; @@ -20,7 +21,7 @@ export const initResult = (newItem) => { }); } -export const addResult = (model, modeltype, datatype, backend, backendstatus, backendinference) => { +export const addResult = (model, modeltype, datatype, backend, status, warmup, inference, inferencemedian) => { resultsStore.update(items => { return items.map(item => { if ( @@ -31,8 +32,10 @@ export const addResult = (model, modeltype, datatype, backend, backendstatus, ba const updatedItem = { ...item }; for (const key in updatedItem) { if (key !== "id" && key !== "model" && key !== "modeltype" && key !== "datatype") { - updatedItem[backend].status = backendstatus; - updatedItem[backend].inference = backendinference; + updatedItem[backend].status = status; + updatedItem[backend].warmup = warmup; + updatedItem[backend].inference = inference; + updatedItem[backend].inferencemedian = inferencemedian; } } return updatedItem; @@ -329,7 +332,7 @@ export const containsAllElementsInArray = (string, array) => { } export const urlToStore = (urlSearchParams, modelIdFromUrl) => { - if (urlSearchParams.size > 0) { + if (urlSearchParams.size > 0 && urlSearchParams.size != 1) { let modelType = urlSearchParams.get('modeltype'); let dataType = urlSearchParams.get('datatype'); let backend = urlSearchParams.get('backend'); @@ -405,27 +408,22 @@ export const random = () => { return (Math.random() * (1000 - 1) + 1).toFixed(2); }; -export const median = (arr, length) => { - if (arr.length == 0) { - return; - } - const sorted = arr.sort((a, b) => a - b); - const middle = Math.floor(sorted.length / 2); +export const median = (arr) => { + const sorted = arr.slice().sort((a, b) => a - b); + + // Check if the array length is even or odd + const middle = Math.floor(sorted.length / 2.0); + if (sorted.length % 2 === 0) { - let evenSum = 0; - if (length === 0) { - evenSum = parseInt(sorted[middle - 1]) + parseInt(sorted[middle]); - } else if (length === 2) { - evenSum = parseFloat(sorted[middle - 1]) + parseFloat(sorted[middle]); - } - return (evenSum / 2.0).toFixed(length); + // If the array length is even, return the average of the two middle values + return (parseFloat(sorted[middle - 1]) + parseFloat(sorted[middle])) / 2.0; } else { - return sorted[middle]; + // If the array length is odd, return the middle value + return parseFloat(sorted[middle].toFixed(2)); } }; const runSingleTest = async (id, model, modeltype, datatype, backend) => { - console.log(`== ${id}, ${model}, ${modeltype}, ${datatype}, ${backend} ==`) updateTestQueueStatus(id, 2); // Test in Progress addResult(model, modeltype, datatype, backend, 1, []); updateInfo(`${testQueueLength - testQueue.length}/${testQueueLength} Testing ${model} (${modeltype}/${datatype}) with ${backend} backend ...`); @@ -454,7 +452,7 @@ export const run = async () => { } initResult(r); - await runSingleTest(t0.id, t0.model, t0.modeltype, t0.datatype, t0.backend); + await catchMain(t0.id, t0.model, t0.modeltype, t0.datatype, t0.backend); filterTestQueue(t0.id); run(); @@ -577,3 +575,8 @@ export const getModelIdfromPath = () => { path = path.replace('/web-ai-benchmark/run/', '').replaceAll('/run/', '').replaceAll('/', '').trim().toLowerCase(); return path; } + +export const catchEm = (promise) => { + return promise.then(data => [null, data]) + .catch(err => [err]); +} \ No newline at end of file diff --git a/src/lib/components/ConfigDataTypes.svelte b/src/lib/components/ConfigDataTypes.svelte index 0336c22..de6c3d6 100644 --- a/src/lib/components/ConfigDataTypes.svelte +++ b/src/lib/components/ConfigDataTypes.svelte @@ -105,7 +105,7 @@
diff --git a/src/lib/components/ConfigModels.svelte b/src/lib/components/ConfigModels.svelte index 92d58ac..8ad43c0 100644 --- a/src/lib/components/ConfigModels.svelte +++ b/src/lib/components/ConfigModels.svelte @@ -350,7 +350,7 @@
diff --git a/src/lib/components/ConfigNumOfRuns.svelte b/src/lib/components/ConfigNumOfRuns.svelte index d99aa52..a3ef600 100644 --- a/src/lib/components/ConfigNumOfRuns.svelte +++ b/src/lib/components/ConfigNumOfRuns.svelte @@ -1,6 +1,7 @@
diff --git a/src/lib/components/Header.svelte b/src/lib/components/Header.svelte index bd5df37..25019f1 100644 --- a/src/lib/components/Header.svelte +++ b/src/lib/components/Header.svelte @@ -45,7 +45,7 @@ if (search.indexOf('?') > -1) { fullSearch = search + '&model=' + getModelIdfromPath(); } else { - fullSearch = search + '?model=' + getModelIdfromPath(); + fullSearch = search; } } else { fullSearch = search; diff --git a/src/lib/components/Nav.svelte b/src/lib/components/Nav.svelte index 5d83ee9..80ea184 100644 --- a/src/lib/components/Nav.svelte +++ b/src/lib/components/Nav.svelte @@ -15,7 +15,7 @@ if (search.indexOf('?') > -1) { fullSearch = search + '&model=' + getModelIdfromPath(); } else { - fullSearch = search + '?model=' + getModelIdfromPath(); + fullSearch = search; } } else { fullSearch = search; @@ -36,7 +36,7 @@ diff --git a/src/lib/components/Results.svelte b/src/lib/components/Results.svelte index d09acba..42eab57 100644 --- a/src/lib/components/Results.svelte +++ b/src/lib/components/Results.svelte @@ -1,7 +1,6 @@