diff --git a/src/lib/assets/css/_vars.css b/src/lib/assets/css/_vars.css index 46b87ff..ed8e2bc 100644 --- a/src/lib/assets/css/_vars.css +++ b/src/lib/assets/css/_vars.css @@ -6,9 +6,11 @@ --red-01: rgba(198, 26, 62, 0.1); --green: rgba(106, 198, 0, 1); --green-01: rgba(106, 198, 0, 0.1); + --green-002: rgba(106, 198, 0, 0.01); --p: rgba(136, 75, 233, 1); --purple: rgba(22, 23, 72, 1); --pink: rgba(249, 93, 155, 1); + --orange: rgba(249, 155, 93, 1); --blue: rgba(0, 132, 255, 1); --blue-01: rgba(0, 132, 255, 0.1); --grey: rgba(213, 213, 213, 1); diff --git a/src/lib/assets/css/global.css b/src/lib/assets/css/global.css index 0b26491..3990b82 100644 --- a/src/lib/assets/css/global.css +++ b/src/lib/assets/css/global.css @@ -201,19 +201,74 @@ header nav { display: grid; grid-template-columns: repeat(3, 1fr); grid-template-rows:auto; - grid-column-gap: 20px; + grid-column-gap: 0px; grid-row-gap: 0px; } -.tq svg { +.tq svg, .result svg { display: inline-block; - height: 13px; +} + +.result .s { + display: flex; + flex-direction: row; + align-items: center; +} + +.tq .s svg, .result .s svg { + height: 16px; + padding-top: 2px; +} + +.tq .s svg path, .result .s svg path { + fill: var(--grey); +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +.q:hover .s svg path { + fill: var(--green); +} + +.result .q:hover .status_1 svg path { + fill: var(--grey) !important; +} + +.result .q:hover .status_2 svg path { + fill: var(--orange) !important; +} + +.result .q:hover .status_3 svg path { + fill: var(--green) !important; +} + +.result .q:hover .status_4 svg path { + fill: var(--red) !important; +} + +.status_2 svg path { + fill: var(--green) !important; + animation: spin 2000ms linear infinite; + transform-box: fill-box; + transform-origin: center; } .tq .onnx svg, .tq .tflite svg, .tq .npy svg, .tq .pt svg { width: 16px; } +.result .onnx svg, .result .tflite svg, .result .npy svg, .result .pt svg { + width: 16px; + margin-bottom: -2px; +} + .tq #logo_onnx { margin: 0 0 -1px 0px; } @@ -230,7 +285,7 @@ header nav { margin: 0 0 -1px 2px; } -.q div { +.tq .q div { display: inline-block; text-align: center; overflow: hidden; @@ -238,21 +293,25 @@ header nav { text-overflow: ellipsis; } -.q { +.tq .q { padding-top: 3px; - margin-bottom: 2px; + margin: 0 -1px -1px 0; border: 1px solid var(--grey-02); + display: grid; + grid-template-columns: repeat(5, 1fr); + grid-template-rows:auto; + grid-column-gap: 0px; + grid-row-gap: 0px; + justify-items: center; } -.q:hover { - border: 1px solid var(--red); -} - -.tq .m, .tq .mt, .tq .dt, .tq .bk { - height: 18px; +.tq .q:hover, .result .q:hover { + background-color: var(--green-002); + border-bottom: 1px solid var(--green); } -.tq .m, .tq .dt, .tq .bk { +.tq .s, .tq .m, .tq .dt, .tq .bk, +.result .s, .result .m, .result .dt, .result .bk { padding: 2px 4px 2px 4px; } @@ -262,15 +321,19 @@ padding: 2px 4px 2px 4px; margin-bottom: -1px; } -.q .m { - width: 200px; +.tq .q .s { + width: 20px; +} + +.tq .q .m { + width: 140px; } -.q .dt { +.tq .q .dt { width: 60px; } -.q .bk { +.tq .q .bk { width: 100px; } @@ -327,6 +390,10 @@ padding: 2px 4px 2px 4px; margin-bottom: -2px; } +.types div { + display: inline-block; +} + .types .true svg#logo_npy .npy-1, .types .true svg#logo_npy .npy-2 { fill: white !important; } @@ -399,4 +466,97 @@ padding: 2px 4px 2px 4px; box-shadow: var(--pink) 0px 6px 12px -8px, var(--pink) 0px 12px 10px -12px; +} + +.result .title { + border: 0px solid transparent !important; + padding: 2px 10px 6px 10px; + border-bottom: 1px solid var(--green) !important; +} + +.result .q div { + text-align: center; + overflow: hidden; + text-overflow: ellipsis; +} + +.result .q { + border-bottom: 1px solid var(--grey-02); + display: grid; + grid-template-rows:auto; + grid-column-gap: 0px; + grid-row-gap: 10px; + justify-items: center; + align-items: center; + padding: 6px; +} + +.result .q._1 { + grid-template-columns: repeat(4, 1fr); +} + +.result .q._2 { + grid-template-columns: repeat(5, 1fr); +} + +.result .q._3 { + grid-template-columns: repeat(6, 1fr); +} + +.result .q._4 { + grid-template-columns: repeat(7, 1fr); +} + +.result .q._5 { + grid-template-columns: repeat(8, 1fr); +} + +.result .q._6 { + grid-template-columns: repeat(9, 1fr); +} + +.result .q._7 { + grid-template-columns: repeat(10, 1fr); +} + +.result .q._8 { + grid-template-columns: repeat(11, 1fr); +} + + +@media (max-width: 1024px) { + .tq { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 500px) { + .tq { + grid-template-columns: repeat(1, 1fr); + } + .group label { + margin-left: 0px; + } + + .group.true { + background-color: transparent; + } + + .group.true label { + background-color: var(--red); + color: var(--white); + } + + .group.true span { + color: var(--font); + display: block; + } + + .types div:last-child { + margin-top: 4px; + } + + .config .models label { + width: 98px; + } } \ No newline at end of file diff --git a/src/lib/assets/js/utils.js b/src/lib/assets/js/utils.js index 9f65a2f..481a515 100644 --- a/src/lib/assets/js/utils.js +++ b/src/lib/assets/js/utils.js @@ -25,8 +25,43 @@ export const getGpu = () => { } } -export const addResult = (result) => { - resultsStore.update((arr) => [...arr, result]); +export const initResult = (newItem) => { + resultsStore.update(items => { + const exists = items.some(item => + item.model === newItem.model && + item.modeltype === newItem.modeltype && + item.datatype === newItem.datatype + ); + + if (!exists) { + return [...items, newItem]; + } + + return items; + }); +} + +export const addResult = (model, modeltype, datatype, backend, backendstatus, backendinference) => { + resultsStore.update(items => { + return items.map(item => { + if ( + item.model === model && + item.modeltype === modeltype && + item.datatype === datatype + ) { + const updatedItem = { ...item }; + for (const key in updatedItem) { + if (key !== "id" && key !== "model" && key !== "modeltype" && key !== "datatype") { + updatedItem[backend].status = backendstatus; + updatedItem[backend].inference = backendinference; + } + } + return updatedItem; + } + return item; + }); + }); + } export const resetResult = () => { @@ -158,6 +193,22 @@ export const goTo = () => { } } +export const filterTestQueue = (id) => { + let filteredTestQueue = testQueue.filter((testQueue) => testQueue.id !== id); + testQueueStore.update(() => filteredTestQueue); +} + +export const updateTestQueueStatus = (id, status) => { + testQueueStore.update(items => { + return items.map(item => { + if (item.id === id) { + return { ...item, status: status }; + } + return item; + }); + }); +} + export const updateTestQueue = () => { /** * @type {string[]} @@ -172,14 +223,13 @@ export const updateTestQueue = () => { const matchedModels = models.filter( (model) => model.id === m && model.format === mt && model.datatype === dt ); - if (matchedModels.length > 0) { - // t = `${mt} ${m} ${dt} ${b}`; // testQueue.push(t); - + // Status: 0 Not selected, 1 Not started, 2 In testing, 3 Completed, 4 Fail or Error let t = { id: id, + status: 1, model: m, modeltype: mt, datatype: dt, @@ -192,7 +242,6 @@ export const updateTestQueue = () => { } } } - testQueueStore.update(() => testQueue); } }; @@ -206,11 +255,6 @@ export const stringToArray = (value) => { return value; }; -export const filterTestQueue = (id) => { - let filteredTestQueue = testQueue.filter((testQueue) => testQueue.id !== id); - testQueueStore.update(() => filteredTestQueue); -} - export const urlToStoreHome = (urlSearchParams) => { if (urlSearchParams.size > 0) { let modelType = urlSearchParams.get('modeltype'); @@ -280,4 +324,28 @@ export const urlToStoreHome = (urlSearchParams) => { } }; -export const sleep = (time) => new Promise(resolve => setTimeout(resolve, time)); \ No newline at end of file +export const sleep = (time) => new Promise(resolve => setTimeout(resolve, time)); + +export const random = () => { + return (Math.random() * (1000 - 1) + 1).toFixed(2); +}; + +export const median = (arr, length) => { + if (arr.length == 0) { + return; + } + const sorted = arr.sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 0) { + let evenSum = 0; + if (length === 0) { + evenSum = parseInt(sorted[middle - 1]) + parseInt(sorted[middle]); + } else if (length === 2) { + evenSum = parseFloat(sorted[middle - 1]) + parseFloat(sorted[middle]); + } + return (evenSum / 2.0).toFixed(length); + } else { + return sorted[middle]; + } +}; \ No newline at end of file diff --git a/src/lib/components/ConfigModels.svelte b/src/lib/components/ConfigModels.svelte index 21f9ab7..282da20 100644 --- a/src/lib/components/ConfigModels.svelte +++ b/src/lib/components/ConfigModels.svelte @@ -246,159 +246,163 @@
- - - - + > + Lite + +
+
+ + +
diff --git a/src/lib/components/ConfigModelsManual.svelte b/src/lib/components/ConfigModelsManual.svelte index 1fcab57..def0c06 100644 --- a/src/lib/components/ConfigModelsManual.svelte +++ b/src/lib/components/ConfigModelsManual.svelte @@ -130,159 +130,163 @@
- - - +
+
+ - + + + +
diff --git a/src/lib/components/ConfigNumOfRuns.svelte b/src/lib/components/ConfigNumOfRuns.svelte index 82379eb..99f7b1e 100644 --- a/src/lib/components/ConfigNumOfRuns.svelte +++ b/src/lib/components/ConfigNumOfRuns.svelte @@ -73,4 +73,10 @@ .numofruns label:hover { background-color: transparent; } + + @media (max-width: 500px) { + input[type='range'] { + width: 39vw; + } + } diff --git a/src/lib/components/Results.svelte b/src/lib/components/Results.svelte index c797e8a..439e31e 100644 --- a/src/lib/components/Results.svelte +++ b/src/lib/components/Results.svelte @@ -1,5 +1,6 @@ -
{JSON.stringify(results)}
+{#if results.length > 0} +
+
+
Model
+
Model Type
+
Data Type
+ {#if selectedBackends && selectedBackends.length > 0} + {#each selectedBackends as backend} +
+ {#if backend === 'wasm_1'} + Wasm 1 thread + {/if} + {#if backend === 'wasm_4'} + Wasm 4 threads + {/if} + {#if backend === 'webgl'} + WebGL + {/if} + {#if backend === 'webgpu'} + WebGPU + {/if} + {#if backend === 'webnn_cpu_1'} + WebNN CPU 1 thread + {/if} + {#if backend === 'webnn_cpu_4'} + WebNN CPU 4 threads + {/if} + {#if backend === 'webnn_gpu'} + WebNN GPU + {/if} + {#if backend === 'webnn_npu'} + WebNN NPU + {/if} + (Median) +
+ {/each} + {/if} +
+ {#each Object.entries(results) as [index, key]} +
+
{key.model}
+ + {#if key.modeltype === 'onnx'} +
+ +
+ {/if} + + {#if key.modeltype === 'tflite'} +
+ +
+ {/if} + + {#if key.modeltype === 'npy'} +
+ +
+ {/if} + + {#if key.modeltype === 'pt'} +
+ +
+ {/if} + +
{key.datatype}
+ + {#if key.wasm_1 && key.wasm_1.status !== 0} + {#if key.wasm_1.status === 1} +
+ +
+ {/if} + + {#if key.wasm_1.status === 2} +
+ +
+ {/if} + + {#if key.wasm_1.status === 3} +
+ + + {median(key.wasm_1.inference, key.wasm_1.inference.length)} + +
+ {/if} + + {#if key.wasm_1.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.wasm_4 && key.wasm_4.status !== 0} + {#if key.wasm_4.status === 1} +
+ +
+ {/if} + + {#if key.wasm_4.status === 2} +
+ +
+ {/if} + + {#if key.wasm_4.status === 3} +
+ + + {median(key.wasm_4.inference, key.wasm_4.inference.length)} + +
+ {/if} + + {#if key.wasm_4.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.webgl && key.webgl.status !== 0} + {#if key.webgl.status === 1} +
+ +
+ {/if} + + {#if key.webgl.status === 2} +
+ +
+ {/if} + + {#if key.webgl.status === 3} +
+ + + {median(key.webgl.inference, key.webgl.inference.length)} + +
+ {/if} + + {#if key.webgl.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.webgpu && key.webgpu.status !== 0} + {#if key.webgpu.status === 1} +
+ +
+ {/if} + + {#if key.webgpu.status === 2} +
+ +
+ {/if} + + {#if key.webgpu.status === 3} +
+ + + {median(key.webgpu.inference, key.webgpu.inference.length)} + +
+ {/if} + + {#if key.webgpu.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.webnn_cpu_1 && key.webnn_cpu_1.status !== 0} + {#if key.webnn_cpu_1.status === 1} +
+ +
+ {/if} + + {#if key.webnn_cpu_1.status === 2} +
+ +
+ {/if} + + {#if key.webnn_cpu_1.status === 3} +
+ + + {median(key.webnn_cpu_1.inference, key.webnn_cpu_1.inference.length)} + +
+ {/if} + + {#if key.webnn_cpu_1.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.webnn_cpu_4 && key.webnn_cpu_4.status !== 0} + {#if key.webnn_cpu_4.status === 1} +
+ +
+ {/if} + + {#if key.webnn_cpu_4.status === 2} +
+ +
+ {/if} + + {#if key.webnn_cpu_4.status === 3} +
+ + + {median(key.webnn_cpu_4.inference, key.webnn_cpu_4.inference.length)} + +
+ {/if} + + {#if key.webnn_cpu_4.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.webnn_gpu && key.webnn_gpu.status !== 0} + {#if key.webnn_gpu.status === 1} +
+ +
+ {/if} + + {#if key.webnn_gpu.status === 2} +
+ +
+ {/if} + + {#if key.webnn_gpu.status === 3} +
+ + + {median(key.webnn_gpu.inference, key.webnn_gpu.inference.length)} + +
+ {/if} + + {#if key.webnn_gpu.status === 4} +
+ +
+ {/if} + {/if} + + {#if key.webnn_npu && key.webnn_npu.status !== 0} + {#if key.webnn_npu.status === 1} +
+ +
+ {/if} + + {#if key.webnn_npu.status === 2} +
+ +
+ {/if} + + {#if key.webnn_npu.status === 3} +
+ + + {median(key.webnn_npu.inference, key.webnn_npu.inference.length)} + +
+ {/if} + + {#if key.webnn_npu.status === 4} +
+ +
+ {/if} + {/if} +
+ {/each} +
+{/if} diff --git a/src/lib/components/TestQueue.svelte b/src/lib/components/TestQueue.svelte index 33d3fef..eb1712f 100644 --- a/src/lib/components/TestQueue.svelte +++ b/src/lib/components/TestQueue.svelte @@ -16,8 +16,48 @@ {/if}
- {#each testQueue as { id, model, modeltype, datatype, backend }, i} + {#each testQueue as { id, status, model, modeltype, datatype, backend }, i}
+ {#if status === 1} +
+ +
+ {/if} + + {#if status === 2} +
+ +
+ {/if} + + {#if status === 3} +
+ +
+ {/if} + + {#if status === 4} +
+ +
+ {/if} +
{model.replaceAll('_', ' ')}
{#if modeltype === 'onnx'}
diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 07073b5..a7082bb 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -9,7 +9,7 @@ import { goto } from '$app/navigation'; import { onDestroy } from 'svelte'; import { autoStore, testQueueStore } from '../lib/store/store'; - import { urlToStoreHome } from '../lib/assets/js/utils'; + import { resetResult, urlToStoreHome, sleep } from '../lib/assets/js/utils'; /** * @type {string[]} @@ -19,8 +19,10 @@ testQueue = value; }); - const runTests = () => { + const run = async () => { autoStore.update(() => true); + resetResult(); + sleep(5000); let path = `${base}/run/${testQueue[0].model}`; goto(path); }; @@ -40,7 +42,7 @@
{#if testQueue[0]} - + {/if}
diff --git a/src/routes/run/mobilenet_v2/+page.svelte b/src/routes/run/mobilenet_v2/+page.svelte index 87afede..f45d4f9 100644 --- a/src/routes/run/mobilenet_v2/+page.svelte +++ b/src/routes/run/mobilenet_v2/+page.svelte @@ -6,8 +6,14 @@ import ConfigModelsManual from '$lib/components/ConfigModelsManual.svelte'; import Results from '$lib/components/Results.svelte'; import TestQueue from '$lib/components/TestQueue.svelte'; - import { autoStore, modelsStore, testQueueStore } from '../../../lib/store/store'; - import { addResult, filterTestQueue, sleep } from '../../../lib/assets/js/utils'; + import { autoStore, modelsStore, backendsStore, testQueueStore } from '../../../lib/store/store'; + import { + initResult, + addResult, + filterTestQueue, + sleep, + random + } from '../../../lib/assets/js/utils'; import { base } from '$app/paths'; import { goto } from '$app/navigation'; @@ -27,6 +33,14 @@ selectedModels = value; }); + /** + * @type {string[]} + */ + let selectedBackends; + backendsStore.subscribe((value) => { + selectedBackends = value; + }); + /** * @type {string[]} */ @@ -40,11 +54,7 @@ */ let info; - const random = () => { - return Math.floor(Math.random() * 2); - }; - - const runTests = async () => { + const run = async () => { if ( testQueue[0] && location.pathname.replace('/web-ai-benchmark/run/', '').replace('/run/', '') === @@ -55,13 +65,33 @@ id: t0.id, model: t0.model, modeltype: t0.modeltype, - datatype: t0.datatype, - backend: t0.backend, - inferencetime: [random(), random(), random(), random(), random()] + datatype: t0.datatype }; + + for (const prop of selectedBackends) { + r[prop] = { + status: 1, + inference: [] + }; + } + + initResult(r); + + info = `${t0.model}-${t0.modeltype}-${t0.datatype}-${t0.backend} - Start`; + await sleep(3000); + addResult(t0.model, t0.modeltype, t0.datatype, t0.backend, 1, []); + + await sleep(5000); + info = `${t0.model}-${t0.modeltype}-${t0.datatype}-${t0.backend} - Testing`; + addResult(t0.model, t0.modeltype, t0.datatype, t0.backend, 2, []); + + await sleep(10000); + + addResult(t0.model, t0.modeltype, t0.datatype, t0.backend, 3, [random(), random(), random()]); + + info = `${t0.model}-${t0.modeltype}-${t0.datatype}-${t0.backend} - Completed`; filterTestQueue(t0.id); - addResult(r); - runTests(); + run(); } else if (testQueue[0]) { console.log(testQueue[0].model); let path = `${base}/run/${testQueue[0].model}`; @@ -69,7 +99,7 @@ await sleep(5000); goto(path); } else { - info = `Test completed`; + info = `All tests completed`; let path = `${base}/`; goto(path); } @@ -79,7 +109,7 @@ onMount(() => { if (auto) { - runTests(); + run(); } }); @@ -97,13 +127,15 @@
{#if selectedModels[0] && !auto} - + {/if}
-
- {info} -
+ {#if info} +
+ {info} +
+ {/if}
diff --git a/src/routes/run/resnet50_v1/+page.svelte b/src/routes/run/resnet50_v1/+page.svelte index 87afede..f45d4f9 100644 --- a/src/routes/run/resnet50_v1/+page.svelte +++ b/src/routes/run/resnet50_v1/+page.svelte @@ -6,8 +6,14 @@ import ConfigModelsManual from '$lib/components/ConfigModelsManual.svelte'; import Results from '$lib/components/Results.svelte'; import TestQueue from '$lib/components/TestQueue.svelte'; - import { autoStore, modelsStore, testQueueStore } from '../../../lib/store/store'; - import { addResult, filterTestQueue, sleep } from '../../../lib/assets/js/utils'; + import { autoStore, modelsStore, backendsStore, testQueueStore } from '../../../lib/store/store'; + import { + initResult, + addResult, + filterTestQueue, + sleep, + random + } from '../../../lib/assets/js/utils'; import { base } from '$app/paths'; import { goto } from '$app/navigation'; @@ -27,6 +33,14 @@ selectedModels = value; }); + /** + * @type {string[]} + */ + let selectedBackends; + backendsStore.subscribe((value) => { + selectedBackends = value; + }); + /** * @type {string[]} */ @@ -40,11 +54,7 @@ */ let info; - const random = () => { - return Math.floor(Math.random() * 2); - }; - - const runTests = async () => { + const run = async () => { if ( testQueue[0] && location.pathname.replace('/web-ai-benchmark/run/', '').replace('/run/', '') === @@ -55,13 +65,33 @@ id: t0.id, model: t0.model, modeltype: t0.modeltype, - datatype: t0.datatype, - backend: t0.backend, - inferencetime: [random(), random(), random(), random(), random()] + datatype: t0.datatype }; + + for (const prop of selectedBackends) { + r[prop] = { + status: 1, + inference: [] + }; + } + + initResult(r); + + info = `${t0.model}-${t0.modeltype}-${t0.datatype}-${t0.backend} - Start`; + await sleep(3000); + addResult(t0.model, t0.modeltype, t0.datatype, t0.backend, 1, []); + + await sleep(5000); + info = `${t0.model}-${t0.modeltype}-${t0.datatype}-${t0.backend} - Testing`; + addResult(t0.model, t0.modeltype, t0.datatype, t0.backend, 2, []); + + await sleep(10000); + + addResult(t0.model, t0.modeltype, t0.datatype, t0.backend, 3, [random(), random(), random()]); + + info = `${t0.model}-${t0.modeltype}-${t0.datatype}-${t0.backend} - Completed`; filterTestQueue(t0.id); - addResult(r); - runTests(); + run(); } else if (testQueue[0]) { console.log(testQueue[0].model); let path = `${base}/run/${testQueue[0].model}`; @@ -69,7 +99,7 @@ await sleep(5000); goto(path); } else { - info = `Test completed`; + info = `All tests completed`; let path = `${base}/`; goto(path); } @@ -79,7 +109,7 @@ onMount(() => { if (auto) { - runTests(); + run(); } }); @@ -97,13 +127,15 @@
{#if selectedModels[0] && !auto} - + {/if}
-
- {info} -
+ {#if info} +
+ {info} +
+ {/if}