Skip to content

Commit

Permalink
ex
Browse files Browse the repository at this point in the history
  • Loading branch information
@fs-eire
fs-eire committed Dec 19, 2024
1 parent dfa3e0c commit a3c16a0
Show file tree
Hide file tree
Showing 4 changed files with 248 additions and 56 deletions.
186 changes: 172 additions & 14 deletions js/web/test/e2e/exports/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,37 +4,195 @@
'use strict';

const path = require('path');
const { spawn } = require('child_process');
const EventEmitter = require('node:events');
const treeKill = require('tree-kill');

/**
* Entry point for package exports tests.
*
* @param {string[]} packagesToInstall
*/
module.exports = async function main() {
module.exports = async function main(PRESERVE) {
console.log('Running exports tests...');

// testcases/nextjs-default
{
const workingDir = path.join(__dirname, 'testcases/nextjs-default');
await runShellCmd('npm install ../../onnxruntime-common.tgz ../../onnxruntime-web.tgz', workingDir);
const wd = path.join(__dirname, 'testcases/nextjs-default');
if (!PRESERVE) {
await runShellCmd('npm install ../../onnxruntime-common.tgz ../../onnxruntime-web.tgz', { wd });
}

const launchBrowserAndRunTests = async (logPrefix, port = 3000) => {
const testResults = [];
const puppeteer = require('puppeteer-core');
let browser;
try {
browser = await puppeteer.launch({
executablePath: 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
browser: 'chrome',
headless: true,
args: ['--enable-features=SharedArrayBuffer', '--no-sandbox', '--disable-setuid-sandbox'],
});

for await (const flags of [
[false, false],
//[false, true],
//[true, false],
//[true, true],
]) {
const [multiThread, proxy] = flags;
console.log(`[${logPrefix}] Running test with multi-thread: ${multiThread}, proxy: ${proxy}...`);

const page = await browser.newPage();
await page.goto(`http://localhost:${port}`);
// wait for the page to load
await page.waitForSelector('#ortstate', { visible: true });
// if multi-thread is enabled, check the checkbox
if (multiThread) {
await page.locator('#cb-mt').click();
}
// if proxy is enabled, check the checkbox
if (proxy) {
await page.locator('#cb-px').click();
}
// click the load model button
await page.locator('#btn-load').click();
// wait for the model to load or fail
await page.waitForFunction("['2','3'].includes(document.getElementById('ortstate').innerText)");
// verify the model is loaded
const modelLoadState = await page.$eval('#ortstate', (el) => el.innerText);
if (modelLoadState !== '2') {
const ortLog = await page.$eval('#ortlog', (el) => el.innerText);
testResults.push({ multiThread, proxy, success: false, message: `Failed to load model: ${ortLog}` });
continue;
}

// click the run test button
await page.locator('#btn-run').click();
// wait for the inference run to complete or fail
await page.waitForFunction("['5','6'].includes(document.getElementById('ortstate').innerText)");
// verify the inference run result
const runState = await page.$eval('#ortstate', (el) => el.innerText);
if (runState !== '5') {
const ortLog = await page.$eval('#ortlog', (el) => el.innerText);
testResults.push({ multiThread, proxy, success: false, message: `Failed to run model: ${ortLog}` });
continue;
}

testResults.push({ multiThread, proxy, success: true });
}

return testResults;
} finally {
console.log(`[${logPrefix}] Closing the browser...`);
// close the browser
if (browser) {
await browser.close();
}
}
};

// test dev mode
{
console.log('Testing Next.js default (dev mode)...');
const npmRunDevEvent = new EventEmitter();
const npmRunDev = runShellCmd('npm run dev', {
wd,
event: npmRunDevEvent,
ready: '✓ Ready in',
ignoreExitCode: true,
});

let testResults;
npmRunDevEvent.on('serverReady', async () => {
try {
testResults = await launchBrowserAndRunTests('default:dev');
} finally {
console.log('Killing the server...');
// kill the server as the last step
npmRunDevEvent.emit('kill');
}
});

await npmRunDev;

console.log('Next.js default test (dev mode) result:', testResults);
if (testResults.some((r) => !r.success)) {
throw new Error('Next.js default test (dev mode) failed.');
}
} // test dev mode

// test prod mode
{
console.log('Testing Next.js default (prod mode)...');
// run 'npm run build'
await runShellCmd('npm run build', { wd });
const npmRunStartEvent = new EventEmitter();
const npmRunStart = runShellCmd('npm run start', {
wd,
event: npmRunStartEvent,
ready: '✓ Ready in',
ignoreExitCode: true,
});

let testResults;
npmRunStartEvent.on('serverReady', async () => {
try {
testResults = await launchBrowserAndRunTests('default:prod');
} finally {
console.log('Killing the server...');
// kill the server as the last step
npmRunStartEvent.emit('kill');
}
});

await npmRunStart;

console.log('Next.js default test (prod mode) result:', testResults);
if (testResults.some((r) => !r.success)) {
throw new Error('Next.js default test (prod mode) failed.');
}
} // test prod mode
}
};

async function runShellCmd(cmd, wd = __dirname) {
async function runShellCmd(cmd, { wd = __dirname, event = null, ready = null, ignoreExitCode = false }) {
console.log('===============================================================');
console.log(' Running command in shell:');
console.log(' > ' + cmd);
console.log('===============================================================');
let complete = false;
const childProcess = spawn(cmd, { shell: true, stdio: 'inherit', cwd: wd });
childProcess.on('close', function (code) {
if (code !== 0) {
process.exit(code);
} else {
complete = true;

return new Promise((resolve, reject) => {
const childProcess = spawn(cmd, { shell: true, stdio: ['ignore', 'pipe', 'inherit'], cwd: wd });
childProcess.on('close', function (code, signal) {
if (code === 0 || ignoreExitCode) {
resolve();
} else {
reject(`Process exits with code ${code}`);
}
});
childProcess.stdout.on('data', (data) => {
process.stdout.write(data);

if (ready && event && data.toString().includes(ready)) {
event.emit('serverReady');
}
});
if (event) {
event.on('kill', () => {
childProcess.stdout.destroy();
treeKill(childProcess.pid);
console.log('killing process...');
});
}
});
while (!complete) {
await delay(100);
}
}

async function delay(ms) {
return new Promise(function (resolve) {
setTimeout(function () {
resolve();
}, ms);
});
}
79 changes: 51 additions & 28 deletions js/web/test/e2e/exports/testcases/nextjs-default/components/onnx-test-bar.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,34 +3,57 @@
import { useState } from 'react';

export default function OnnxTestBar() {
const [ortState, setOrtState] = useState('');
const [ortState, setOrtState] = useState(0);
const [ortLog, setOrtLog] = useState('Ready.');

const loadModel = async () => {
setOrtState('Loading model...');
try {
const { createTestSession } = await import('../app/onnx-helper');
await createTestSession(document.getElementById('cb-mt').checked, document.getElementById('cb-px').checked);
} catch (e) {
setOrtState(`Failed to load model: ${e}`);
return;
}
setOrtState('Model loaded.');
};
const loadModel = async () => {
setOrtState(1);
setOrtLog('Loading model...');
try {
const { createTestSession } = await import('../app/onnx-helper');
await createTestSession(document.getElementById('cb-mt').checked, document.getElementById('cb-px').checked);
setOrtState(2);
setOrtLog('Model loaded.');
} catch (e) {
setOrtState(3);
setOrtLog(`Failed to load model: ${e}`);
return;
}
};

const runTest = async () => {
setOrtState('Running model test...');
const { runTestSessionAndValidate } = await import('../app/onnx-helper');
const testResult = await runTestSessionAndValidate();
setOrtState(`Test result: ${testResult}`);
};
const runTest = async () => {
setOrtState(4);
setOrtLog('Running model test...');
try {
const { runTestSessionAndValidate } = await import('../app/onnx-helper');
const testResult = await runTestSessionAndValidate();
setOrtState(5);
setOrtLog(`Test result: ${testResult}`);
} catch (e) {
setOrtState(6);
setOrtLog(`Failed to load model: ${e}`);
return;
}
};

return (
<div>
<label><input type="checkbox" title="Multi-thread" id="cb-mt" />Multi-thread</label>
<label><input type="checkbox" title='Proxy' id="cb-px" />Proxy</label>
<button onClick={loadModel}>Load Model</button>
<button onClick={runTest}>Run Test</button>
<div>{ortState}</div>
</div>
);
}
return (
<div>
<label>
<input type="checkbox" title="Multi-thread" id="cb-mt" />
Multi-thread
</label>
<label>
<input type="checkbox" title="Proxy" id="cb-px" />
Proxy
</label>
<button id="btn-load" onClick={loadModel} disabled={ortState !== 0}>
Load Model
</button>
<button id="btn-run" onClick={runTest} disabled={ortState !== 2}>
Run Test
</button>
<div id="ortstate">{ortState}</div>
<div id="ortlog">{ortLog}</div>
</div>
);
}
2 changes: 2 additions & 0 deletions js/web/test/e2e/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@
"mocha": "^10.4.0",
"parcel": "^2.12.0",
"parcel-reporter-static-files-copy": "^1.5.3",
"puppeteer-core": "23.11.0",
"rollup": "^4.13.2",
"rollup-plugin-copy": "^3.5.0",
"tree-kill": "^1.2.2",
"webpack-cli": "^5.1.4"
},
"scripts": {
Expand Down
37 changes: 23 additions & 14 deletions js/web/test/e2e/run.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,15 @@ const minimist = require('minimist');

const { NODEJS_TEST_CASES, BROWSER_TEST_CASES, BUNDLER_TEST_CASES } = require('./run-data');

// commandline arguments
const parsedArgs = minimist(process.argv.slice(2));
const BROWSER = parsedArgs.browser || 'Chrome_default';
// Preserve the existing test folder
// When this flag is set, the test folder will not be cleared before running the tests.
// NPM install will not be run again.
// This is useful for debugging test failures as it makes it much faster to re-run.
const PRESERVE = parsedArgs.preserve || parsedArgs.p;

// copy whole folder to out-side of <ORT_ROOT>/js/ because we need to test in a folder that no `package.json` file
// exists in its parent folder.
// here we use <ORT_ROOT>/build/js/e2e/ for the test
Expand All @@ -20,9 +29,11 @@ const JS_ROOT_FOLDER = path.resolve(__dirname, '../../..');
const TEST_E2E_RUN_FOLDER = path.resolve(JS_ROOT_FOLDER, '../build/js/e2e');
const NPM_CACHE_FOLDER = path.resolve(TEST_E2E_RUN_FOLDER, '../npm_cache');
const CHROME_USER_DATA_FOLDER = path.resolve(TEST_E2E_RUN_FOLDER, '../user_data');
fs.emptyDirSync(TEST_E2E_RUN_FOLDER);
fs.emptyDirSync(NPM_CACHE_FOLDER);
fs.emptyDirSync(CHROME_USER_DATA_FOLDER);
if (!PRESERVE) {
fs.emptyDirSync(TEST_E2E_RUN_FOLDER);
fs.emptyDirSync(NPM_CACHE_FOLDER);
fs.emptyDirSync(CHROME_USER_DATA_FOLDER);
}
fs.copySync(TEST_E2E_SRC_FOLDER, TEST_E2E_RUN_FOLDER);

// always use a new folder as user-data-dir
Expand All @@ -34,9 +45,6 @@ function getNextUserDataDir() {
return dir;
}

// commandline arguments
const BROWSER = minimist(process.argv.slice(2)).browser || 'Chrome_default';

async function main() {
// find packed package
const { globbySync } = await import('globby');
Expand All @@ -61,21 +69,22 @@ async function main() {

// we start here:

if (!PRESERVE) {
// install dev dependencies
await runInShell(`npm install`);

// npm install with "--cache" to install packed packages with an empty cache folder
await runInShell(`npm install --cache "${NPM_CACHE_FOLDER}" ${PACKAGES_TO_INSTALL.map((i) => `"${i}"`).join(' ')}`);
}

// perform exports testing
{
fs.copySync(PACKAGES_TO_INSTALL[0], path.join(TEST_E2E_RUN_FOLDER, 'exports', 'onnxruntime-common.tgz'));
fs.copySync(PACKAGES_TO_INSTALL[1], path.join(TEST_E2E_RUN_FOLDER, 'exports', 'onnxruntime-web.tgz'));
const testExportsMain = require(path.join(TEST_E2E_RUN_FOLDER, './exports/main'));
await testExportsMain();
process.exit(0);
await testExportsMain(PRESERVE);
}

// install dev dependencies
await runInShell(`npm install`);

// npm install with "--cache" to install packed packages with an empty cache folder
await runInShell(`npm install --cache "${NPM_CACHE_FOLDER}" ${PACKAGES_TO_INSTALL.map((i) => `"${i}"`).join(' ')}`);

// prepare .wasm files for path override testing
prepareWasmPathOverrideFiles();

Expand Down

0 comments on commit a3c16a0

Please sign in to comment.