From 0313dd1f65a329d001c32b1a6ba45bc52d0f16fa Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:16:59 -0700 Subject: [PATCH] Update Web CI to use data dir under Agent.TempDirectory (#20074) ### Description Update Web CI to use data dir under Agent.TempDirectory This change fixes the random failure caused by unstable access to karma temp directory (which is under AppData\Local\Temp) on CI pipeline --- js/web/karma.conf.js | 9 ++-- js/web/script/test-runner-cli-args.ts | 6 +++ js/web/script/test-runner-cli.ts | 5 +- .../azure-pipelines/templates/win-web-ci.yml | 50 +++++++++++++++---- .../templates/win-web-multi-browsers.yml | 21 ++++++-- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/js/web/karma.conf.js b/js/web/karma.conf.js index 9e44d9c0d9652..507da0de2b4ad 100644 --- a/js/web/karma.conf.js +++ b/js/web/karma.conf.js @@ -9,6 +9,8 @@ const karmaPlugins = args['karma-plugins'] || undefined; const timeoutMocha = args['timeout-mocha'] || 60000; const forceLocalHost = !!args['force-localhost']; +// user data directory; will be passed to the Edge/Chrome/ChromeCanary/Firefox launchers +const userDataDir = args['user-data-dir']; // parse chromium flags let chromiumFlags = args['chromium-flags']; if (!chromiumFlags) { @@ -87,9 +89,10 @@ module.exports = function(config) { listenAddress, customLaunchers: { // Chromium-based browsers - EdgeTest: {base: 'Edge', flags: chromiumFlags}, - ChromeTest: {base: 'Chrome', flags: chromiumFlags}, - ChromeCanaryTest: {base: 'ChromeCanary', flags: chromiumFlags}, + EdgeTest: {base: 'Edge', flags: chromiumFlags, edgeDataDir: userDataDir}, + ChromeTest: {base: 'Chrome', flags: chromiumFlags, chromeDataDir: userDataDir}, + ChromeCanaryTest: {base: 'ChromeCanary', flags: chromiumFlags, chromeDataDir: userDataDir}, + FirefoxTest: {base: 'Firefox', profile: userDataDir}, // // ==== BrowserStack browsers ==== diff --git a/js/web/script/test-runner-cli-args.ts b/js/web/script/test-runner-cli-args.ts index b2b212bdb9bc1..745f504b0494d 100644 --- a/js/web/script/test-runner-cli-args.ts +++ b/js/web/script/test-runner-cli-args.ts @@ -103,6 +103,7 @@ Options: --no-sandbox This flag will be passed to Chrome. Sometimes Chrome need this flag to work together with Karma. + --user-data-dir=<...> This flag will be passed to browsers to specify the user data directory. --chromium-flags=<...> This flag will be passed to Chrome and Edge browsers. Can be used multiple times. Examples: @@ -195,6 +196,7 @@ export interface TestRunnerCliArgs { webnnOptions?: InferenceSession.WebNNExecutionProviderOption; globalEnvFlags?: Test.Options['globalEnvFlags']; noSandbox?: boolean; + userDataDir?: string; chromiumFlags: string[]; } @@ -477,6 +479,9 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs // Option: --no-sandbox const noSandbox = !!args['no-sandbox']; + // Option: --user-data-dir + const userDataDir = args['user-data-dir']; + // parse chromium flags let chromiumFlags = args['chromium-flags']; if (!chromiumFlags) { @@ -515,6 +520,7 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs wasmOptions, globalEnvFlags, noSandbox, + userDataDir, chromiumFlags }; } diff --git a/js/web/script/test-runner-cli.ts b/js/web/script/test-runner-cli.ts index ace64e9532b12..03d637b35bc7c 100644 --- a/js/web/script/test-runner-cli.ts +++ b/js/web/script/test-runner-cli.ts @@ -573,6 +573,9 @@ async function main() { karmaArgs.push('--log-level debug'); } karmaArgs.push(`--bundle-mode=${args.bundleMode}`); + if (args.userDataDir) { + karmaArgs.push(`--user-data-dir="${args.userDataDir}"`); + } karmaArgs.push(...chromiumFlags.map(flag => `--chromium-flags=${flag}`)); if (browser.startsWith('Edge')) { // There are currently 2 Edge browser launchers: @@ -671,7 +674,7 @@ async function main() { case 'edge': return 'EdgeTest'; case 'firefox': - return 'Firefox'; + return 'FirefoxTest'; case 'electron': return 'Electron'; case 'safari': diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml index 1eb2ee6f6409c..b7aee559cf73c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml @@ -153,31 +153,61 @@ jobs: errorActionPreference: stop displayName: 'Pack NPM packages' - script: | - npm test -- -e=chrome -b=webgl,wasm --karma-debug + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\01 + npm test -- -e=chrome -b=webgl,wasm --user-data-dir=$(Agent.TempDirectory)\web\test\01 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (wasm,webgl backend)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'false') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'false')) + - script: | + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) - script: | - npm test -- -e=chrome -b=webgl,wasm,webgpu --karma-debug $(webgpuCommandlineExtraFlags) + mkdir $(Agent.TempDirectory)\web\test\02 + npm test -- -e=chrome -b=webgl,wasm,webgpu $(webgpuCommandlineExtraFlags) --user-data-dir=$(Agent.TempDirectory)\web\test\02 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (ALL backends)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'true') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'true')) + - script: | + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) - script: | - npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-tensor --karma-debug $(webgpuCommandlineExtraFlags) + mkdir $(Agent.TempDirectory)\web\test\03 + npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-tensor $(webgpuCommandlineExtraFlags) --user-data-dir=$(Agent.TempDirectory)\web\test\03 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (Suite1, webgpu, IO-binding=gpu-tensor)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'true') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'true')) - script: | - npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-location --karma-debug $(webgpuCommandlineExtraFlags) + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\04 + npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-location $(webgpuCommandlineExtraFlags) --user-data-dir=$(Agent.TempDirectory)\web\test\04 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (Suite1, webgpu, IO-binding=gpu-location)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'true') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'true')) + - script: | + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) - script: | - npm test -- --webgl.pack -b=webgl -e=chrome --karma-debug + mkdir $(Agent.TempDirectory)\web\test\05 + npm test -- --webgl.pack -b=webgl -e=chrome --user-data-dir=$(Agent.TempDirectory)\web\test\05 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebGL: packed mode' - script: | - npm test -- --wasm.proxy -b=wasm -e=chrome --karma-debug + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\06 + npm test -- --wasm.proxy -b=wasm -e=chrome --user-data-dir=$(Agent.TempDirectory)\web\test\06 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebAssembly: proxy' condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml index 79bf0b5e71363..00109b348e8cb 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml @@ -68,15 +68,30 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm ci /js/web/' - script: | - npm test -- suite0 -b=wasm,webgl --wasm.initTimeout=30000 --file-cache + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test_multi_browsers\01 + npm test -- suite0 -e=chrome -b=wasm,webgl --wasm.initTimeout=30000 --file-cache --user-data-dir=$(Agent.TempDirectory)\web\test_multi_browsers\01 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Chrome)' - script: | - npm test -- suite0 -b=wasm,webgl -e=firefox --wasm.initTimeout=30000 --file-cache + powershell "Get-WmiObject Win32_Process -Filter \"name = 'firefox.exe'\" | Format-List CommandLine" + displayName: 'Check active Firefox processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test_multi_browsers\02 + npm test -- suite0 -b=wasm,webgl -e=firefox --wasm.initTimeout=30000 --file-cache --user-data-dir=$(Agent.TempDirectory)\web\test_multi_browsers\02 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Firefox)' - script: | - npm test -- suite0 -b=wasm,webgl -e=edge --wasm.initTimeout=30000 --file-cache + powershell "Get-WmiObject Win32_Process -Filter \"name = 'msedge.exe'\" | Format-List CommandLine" + displayName: 'Check active Edge processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test_multi_browsers\03 + npm test -- suite0 -b=wasm,webgl -e=edge --wasm.initTimeout=30000 --file-cache --user-data-dir=$(Agent.TempDirectory)\web\test_multi_browsers\03 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Edge)' - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3