diff --git a/.github/workflows/build_test_dml.yml b/.github/workflows/build_test_dml.yml new file mode 100644 index 000000000..81da4670d --- /dev/null +++ b/.github/workflows/build_test_dml.yml @@ -0,0 +1,123 @@ +name: DirectML backend (Windows) + +on: [push, pull_request] + +jobs: + + job: + + runs-on: windows-2019 + + steps: + - name: Git config + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Install depot_tools + shell: cmd + run: | + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools + set "PATH=%CD%\..\depot_tools;%PATH%" + gclient + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - uses: actions/checkout@v2 + with: + ref: main + path: baseline + fetch-depth: 0 + + - name: Update DEPS for main branch + shell: pwsh + run: | + cd baseline + (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS + (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS + + - name: Sync code for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd baseline + copy scripts\standalone.gclient .gclient + gclient sync + + - name: Generate project for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd baseline + gn gen out\Release --args="webnn_enable_dml=true is_debug=false gpgmm_enable_device_checks=true" + + - name: Build for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd baseline + ninja -C out\Release + + - name: Test for main branch + shell: cmd + run: | + cd baseline + echo "Run End2End Tests..." + out\Release\webnn_end2end_tests.exe --gtest_output=json:${{ github.workspace }}\..\baseline_end2endtests.json + cd .. + rmdir /s /q baseline + + - uses: actions/checkout@v2 + with: + path: update + fetch-depth: 0 + + - name: Update DEPS for update branch + shell: pwsh + run: | + cd update + (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS + (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS + + - name: Sync latest code + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + copy scripts\standalone.gclient .gclient + gclient sync + + - name: Generate project for update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + gn gen out\Release --args="webnn_enable_dml=true webnn_enable_wire=true is_debug=false gpgmm_enable_device_checks=true" + + - name: Build for update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + ninja -C out\Release + + - name: Test for update branch + shell: cmd + run: | + cd update + echo "Run End2End Tests..." + out\Release\webnn_end2end_tests.exe --gtest_output=json:${{ github.workspace }}\..\update_end2endtests.json || true + + - name: Regression check + run: | + echo "Regression checking..." + python update\workflow_scripts\regression_check.py ${{ github.workspace }}\..\baseline_end2endtests.json ${{ github.workspace }}\..\update_end2endtests.json diff --git a/.github/workflows/build_test_node_dml.yml b/.github/workflows/build_test_node_dml.yml new file mode 100644 index 000000000..f776879c1 --- /dev/null +++ b/.github/workflows/build_test_node_dml.yml @@ -0,0 +1,168 @@ +name: Node Binding (DirectML backend / Windows) + +on: [push, pull_request] + +jobs: + + job: + + runs-on: windows-2019 + + steps: + - name: Git config + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Install depot_tools + shell: cmd + run: | + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools + set "PATH=%CD%\..\depot_tools;%PATH%" + gclient + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - uses: actions/setup-node@v2 + with: + node-version: '14' + + - uses: actions/checkout@v2 + with: + ref: main + path: baseline + fetch-depth: 0 + + - name: Update DEPS for main branch + shell: pwsh + run: | + cd baseline + (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS + (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS + + - name: Sync code for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd baseline + copy scripts\standalone.gclient .gclient + gclient sync + + - name: Generate project for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd baseline + gn gen out\Release --args="webnn_enable_dml=true is_debug=false gpgmm_enable_device_checks=true" + + - name: Build for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd baseline + ninja -C out\Release + + - name: Run 'npm install' command under node folder of main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%CD%\baseline\out\Release;%PATH%" + cd baseline\node + npm install --webnn_native_lib_path="../out/Release" + + - name: Run 'npm run build' command under node folder of main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%CD%\baseline\out\Release;%PATH%" + cd baseline\node + npm run build --webnn_native_lib_path="../out/Release" + + - name: Run 'npm run report' command under node folder of main branch + shell: cmd + run: | + set "PATH=%CD%\baseline\out\Release;%PATH%" + cd baseline\node + npm run report || true + + - name: Prepare baseline result file for regression checking + shell: cmd + run: | + echo "Baseline node test result:" + type baseline\node\result.xml + copy baseline\node\result.xml ${{ github.workspace }}\..\baseline.xml + rmdir /s /q baseline + + - uses: actions/checkout@v2 + with: + path: update + fetch-depth: 0 + + - name: Update DEPS for update branch + shell: pwsh + run: | + cd update + (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS + (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS + + - name: Sync latest code + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + copy scripts\standalone.gclient .gclient + gclient sync + + - name: Generate project for update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + gn gen out\Release --args="webnn_enable_dml=true is_debug=false gpgmm_enable_device_checks=true" + + - name: Build for update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + ninja -C out\Release + + - name: Run 'npm install' command under node folder of update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%CD%\update\out\Release;%PATH%" + cd update\node + npm install --webnn_native_lib_path="../out/Release" + + - name: Run 'npm run build' command under node folder of update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%CD%\update\out\Release;%PATH%" + cd update\node + npm run build --webnn_native_lib_path="../out/Release" + + - name: Run 'npm run report' command under node folder of update branch + shell: cmd + run: | + set "PATH=%CD%\update\out\Release;%PATH%" + cd update\node + npm run report || true + + - name: Prepare latest result file for regression checking + shell: cmd + run: | + echo "Latest node test result:" + type update\node\result.xml + copy update\node\result.xml ${{ github.workspace }}\..\update.xml + + - name: Regression check + run: | + echo "Regression checking..." + python update\workflow_scripts\regression_check.py ${{ github.workspace }}\..\baseline.xml ${{ github.workspace }}\..\update.xml \ No newline at end of file diff --git a/.github/workflows/memory_leak_check_dml.yml b/.github/workflows/memory_leak_check_dml.yml new file mode 100644 index 000000000..6ae2db549 --- /dev/null +++ b/.github/workflows/memory_leak_check_dml.yml @@ -0,0 +1,60 @@ +name: Check memory leak for DirectML backend (Windows) + +on: [push, pull_request] + +jobs: + + job: + + runs-on: windows-2019 + + steps: + - name: Git config + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Install depot_tools + shell: cmd + run: | + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools + set "PATH=%CD%\..\depot_tools;%PATH%" + gclient + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - uses: actions/checkout@v2 + with: + path: update + fetch-depth: 0 + + - name: Sync latest code + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + copy scripts\standalone.gclient .gclient + gclient sync + - name: Generate project for update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + gn gen out\Debug --args="webnn_enable_dml=true is_debug=true" + - name: Build for update branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd update + ninja -C out\Debug + - name: Check memory leak for update branch + shell: cmd + run: | + cd update + echo "Run End2End Tests..." + out\Debug\webnn_end2end_tests.exe --gtest_filter=-GemmTests.ScalarBias:Pool2dTests.MaxPool2dDilationsDefault:Pool2dTests.MaxPool2dDilationsNhwc:Pool2dTests.*Pool2dAutoPadExplicitOutputSizes4x4Nhwc:Pool2dTests.*Pool2dAutoPadExplicitRoundingTypeCeilNhwc > update_end2endtests.txt || true + python workflow_scripts\memory_leak_check.py update_end2endtests.txt \ No newline at end of file diff --git a/src/webnn/tests/BUILD.gn b/src/webnn/tests/BUILD.gn index bd8c5e63d..815e6bdfa 100644 --- a/src/webnn/tests/BUILD.gn +++ b/src/webnn/tests/BUILD.gn @@ -17,6 +17,7 @@ import("../../../scripts/webnn_overrides_with_defaults.gni") import("//testing/test.gni") import("${webnn_dawn_root}/scripts/dawn_features.gni") +import("${webnn_root}/build_overrides/webnn_features.gni") import("${webnn_root}/generator/webnn_generator.gni") group("webnn_tests") { @@ -199,61 +200,66 @@ source_set("webnn_end2end_tests_sources") { ] sources = [ - "${webnn_root}/examples/MobileNetV2/MobileNetV2.cpp", - "${webnn_root}/examples/MobileNetV2/MobileNetV2.h", - "${webnn_root}/examples/ResNet/ResNet.cpp", - "${webnn_root}/examples/ResNet/ResNet.h", - "${webnn_root}/examples/SqueezeNet/SqueezeNet.cpp", - "${webnn_root}/examples/SqueezeNet/SqueezeNet.h", - "${webnn_root}/examples/SuperResolution/SuperResolution.cpp", - "${webnn_root}/examples/SuperResolution/SuperResolution.h", "WebnnTest.cpp", "WebnnTest.h", - "end2end/AddTests.cpp", - "end2end/BatchNormTests.cpp", - "end2end/ClampTests.cpp", - "end2end/ConcatTests.cpp", - "end2end/Conv2dTests.cpp", - "end2end/ConvTranspose2dTests.cpp", - "end2end/DivTests.cpp", - "end2end/ElementWiseUnaryTests.cpp", - "end2end/GemmTests.cpp", - "end2end/GruTests.cpp", - "end2end/HardSwishTests.cpp", - "end2end/InstanceNormTests.cpp", - "end2end/LeakyReluTests.cpp", - "end2end/MatMulTests.cpp", - "end2end/MaxTests.cpp", - "end2end/MinTests.cpp", - "end2end/MulTests.cpp", - "end2end/PadTests.cpp", - "end2end/Pool2dTests.cpp", - "end2end/PowTests.cpp", - "end2end/ReduceTests.cpp", - "end2end/ReluTests.cpp", - "end2end/Resample2dTests.cpp", - "end2end/ReshapeTests.cpp", - "end2end/SigmoidTests.cpp", - "end2end/SliceTests.cpp", - "end2end/SoftmaxTests.cpp", - "end2end/SplitTests.cpp", - "end2end/SqueezeTests.cpp", - "end2end/SubTests.cpp", - "end2end/TanhTests.cpp", - - # Disable to test unimplemented Sub. - #"end2end/SubTests.cpp", - "end2end/TransposeTests.cpp", - "end2end/models/MobileNetV2BatchNormNchw.cpp", - "end2end/models/MobileNetV2Nchw.cpp", - "end2end/models/MobileNetV2Nhwc.cpp", - "end2end/models/ResNetNchw.cpp", - "end2end/models/ResNetNhwc.cpp", - "end2end/models/SqueezeNetNchw.cpp", - "end2end/models/SqueezeNetNhwc.cpp", - "end2end/models/SuperResolutionNchw.cpp", ] + if (webnn_enable_dml == false) { + sources += [ + "${webnn_root}/examples/MobileNetV2/MobileNetV2.cpp", + "${webnn_root}/examples/MobileNetV2/MobileNetV2.h", + "${webnn_root}/examples/ResNet/ResNet.cpp", + "${webnn_root}/examples/ResNet/ResNet.h", + "${webnn_root}/examples/SqueezeNet/SqueezeNet.cpp", + "${webnn_root}/examples/SqueezeNet/SqueezeNet.h", + "${webnn_root}/examples/SuperResolution/SuperResolution.cpp", + "${webnn_root}/examples/SuperResolution/SuperResolution.h", + "end2end/AddTests.cpp", + "end2end/BatchNormTests.cpp", + "end2end/ClampTests.cpp", + "end2end/ConcatTests.cpp", + "end2end/Conv2dTests.cpp", + "end2end/ConvTranspose2dTests.cpp", + "end2end/DivTests.cpp", + "end2end/ElementWiseUnaryTests.cpp", + "end2end/GemmTests.cpp", + "end2end/GruTests.cpp", + "end2end/HardSwishTests.cpp", + "end2end/InstanceNormTests.cpp", + "end2end/LeakyReluTests.cpp", + "end2end/MatMulTests.cpp", + "end2end/MaxTests.cpp", + "end2end/MinTests.cpp", + "end2end/MulTests.cpp", + "end2end/PadTests.cpp", + "end2end/Pool2dTests.cpp", + "end2end/PowTests.cpp", + "end2end/ReduceTests.cpp", + "end2end/ReluTests.cpp", + "end2end/Resample2dTests.cpp", + "end2end/ReshapeTests.cpp", + "end2end/SigmoidTests.cpp", + "end2end/SliceTests.cpp", + "end2end/SoftmaxTests.cpp", + "end2end/SplitTests.cpp", + "end2end/SqueezeTests.cpp", + "end2end/SubTests.cpp", + "end2end/TanhTests.cpp", + + # Disable to test unimplemented Sub. + #"end2end/SubTests.cpp", + "end2end/TransposeTests.cpp", + "end2end/models/MobileNetV2BatchNormNchw.cpp", + "end2end/models/MobileNetV2Nchw.cpp", + "end2end/models/MobileNetV2Nhwc.cpp", + "end2end/models/ResNetNchw.cpp", + "end2end/models/ResNetNhwc.cpp", + "end2end/models/SqueezeNetNchw.cpp", + "end2end/models/SqueezeNetNhwc.cpp", + "end2end/models/SuperResolutionNchw.cpp", + ] + } + # Validation tests that need OS windows live in end2end tests. libs = []