Add CI and disable end2end tests for DML backend

webmachinelearning · Aug 16, 2022 · fee42f5 · fee42f5
1 parent 95d84e0
commit fee42f5
Show file tree

Hide file tree

Showing 4 changed files with 408 additions and 51 deletions.
diff --git a/.github/workflows/build_test_dml.yml b/.github/workflows/build_test_dml.yml
@@ -0,0 +1,123 @@
+name: DirectML backend (Windows)
+
+on: [push, pull_request]
+
+jobs:
+
+  job:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Git config
+      run: |
+        git config --global core.autocrlf false
+        git config --global core.eol lf
+
+    - name: Install depot_tools
+      shell: cmd
+      run: |
+        git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        gclient
+
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - uses: actions/checkout@v2
+      with:
+        ref: main
+        path: baseline
+        fetch-depth: 0
+
+    - name: Update DEPS for main branch
+      shell: pwsh
+      run: |
+        cd baseline
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS
+
+    - name: Sync code for main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd baseline
+        copy scripts\standalone.gclient .gclient
+        gclient sync
+
+    - name: Generate project for main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd baseline
+        gn gen out\Release --args="webnn_enable_dml=true is_debug=false gpgmm_enable_device_checks=true"
+
+    - name: Build for main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd baseline
+        ninja -C out\Release
+
+    - name: Test for main branch
+      shell: cmd
+      run: |
+        cd baseline
+        echo "Run End2End Tests..."
+        out\Release\webnn_end2end_tests.exe --gtest_output=json:${{ github.workspace }}\..\baseline_end2endtests.json
+        cd ..
+        rmdir /s /q baseline
+
+    - uses: actions/checkout@v2
+      with:
+        path: update
+        fetch-depth: 0
+
+    - name: Update DEPS for update branch
+      shell: pwsh
+      run: |
+        cd update
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS
+
+    - name: Sync latest code
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        copy scripts\standalone.gclient .gclient
+        gclient sync
+
+    - name: Generate project for update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        gn gen out\Release --args="webnn_enable_dml=true webnn_enable_wire=true is_debug=false gpgmm_enable_device_checks=true"
+
+    - name: Build for update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        ninja -C out\Release
+
+    - name: Test for update branch
+      shell: cmd
+      run: |
+        cd update
+        echo "Run End2End Tests..."
+        out\Release\webnn_end2end_tests.exe --gtest_output=json:${{ github.workspace }}\..\update_end2endtests.json || true
+
+    - name: Regression check
+      run: |
+        echo "Regression checking..."
+        python update\workflow_scripts\regression_check.py ${{ github.workspace }}\..\baseline_end2endtests.json ${{ github.workspace }}\..\update_end2endtests.json
diff --git a/.github/workflows/build_test_node_dml.yml b/.github/workflows/build_test_node_dml.yml
@@ -0,0 +1,168 @@
+name: Node Binding (DirectML backend / Windows)
+
+on: [push, pull_request]
+
+jobs:
+
+  job:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Git config
+      run: |
+        git config --global core.autocrlf false
+        git config --global core.eol lf
+
+    - name: Install depot_tools
+      shell: cmd
+      run: |
+        git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        gclient
+
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - uses: actions/setup-node@v2
+      with:
+        node-version: '14'
+
+    - uses: actions/checkout@v2
+      with:
+        ref: main
+        path: baseline
+        fetch-depth: 0
+
+    - name: Update DEPS for main branch
+      shell: pwsh
+      run: |
+        cd baseline
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS
+
+    - name: Sync code for main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd baseline
+        copy scripts\standalone.gclient .gclient
+        gclient sync
+
+    - name: Generate project for main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd baseline
+        gn gen out\Release --args="webnn_enable_dml=true is_debug=false gpgmm_enable_device_checks=true"
+
+    - name: Build for main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd baseline
+        ninja -C out\Release
+
+    - name: Run 'npm install' command under node folder of main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%CD%\baseline\out\Release;%PATH%"
+        cd baseline\node
+        npm install --webnn_native_lib_path="../out/Release"
+
+    - name: Run 'npm run build' command under node folder of main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%CD%\baseline\out\Release;%PATH%"
+        cd baseline\node
+        npm run build --webnn_native_lib_path="../out/Release"
+
+    - name: Run 'npm run report' command under node folder of main branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\baseline\out\Release;%PATH%"
+        cd baseline\node
+        npm run report || true
+
+    - name: Prepare baseline result file for regression checking
+      shell: cmd
+      run: |
+        echo "Baseline node test result:"
+        type baseline\node\result.xml
+        copy baseline\node\result.xml ${{ github.workspace }}\..\baseline.xml
+        rmdir /s /q baseline
+
+    - uses: actions/checkout@v2
+      with:
+        path: update
+        fetch-depth: 0
+
+    - name: Update DEPS for update branch
+      shell: pwsh
+      run: |
+        cd update
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_onnxruntime': True", "'checkout_onnxruntime': False" | Set-Content -path .\DEPS
+        (Get-Content -path .\DEPS -Raw) -replace "'checkout_samples': True", "'checkout_samples': False" | Set-Content -path .\DEPS
+
+    - name: Sync latest code
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        copy scripts\standalone.gclient .gclient
+        gclient sync
+
+    - name: Generate project for update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        gn gen out\Release --args="webnn_enable_dml=true is_debug=false gpgmm_enable_device_checks=true"
+
+    - name: Build for update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        ninja -C out\Release
+
+    - name: Run 'npm install' command under node folder of update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%CD%\update\out\Release;%PATH%"
+        cd update\node
+        npm install --webnn_native_lib_path="../out/Release"
+
+    - name: Run 'npm run build' command under node folder of update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%CD%\update\out\Release;%PATH%"
+        cd update\node
+        npm run build --webnn_native_lib_path="../out/Release"
+
+    - name: Run 'npm run report' command under node folder of update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\update\out\Release;%PATH%"
+        cd update\node
+        npm run report || true
+
+    - name: Prepare latest result file for regression checking
+      shell: cmd
+      run: |
+        echo "Latest node test result:"
+        type update\node\result.xml
+        copy update\node\result.xml ${{ github.workspace }}\..\update.xml
+
+    - name: Regression check
+      run: |
+        echo "Regression checking..."
+        python update\workflow_scripts\regression_check.py ${{ github.workspace }}\..\baseline.xml ${{ github.workspace }}\..\update.xml
diff --git a/.github/workflows/memory_leak_check_dml.yml b/.github/workflows/memory_leak_check_dml.yml
@@ -0,0 +1,60 @@
+name: Check memory leak for DirectML backend (Windows)
+
+on: [push, pull_request]
+
+jobs:
+
+  job:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Git config
+      run: |
+        git config --global core.autocrlf false
+        git config --global core.eol lf
+    - name: Install depot_tools
+      shell: cmd
+      run: |
+        git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        gclient
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - uses: actions/checkout@v2
+      with:
+        path: update
+        fetch-depth: 0
+
+    - name: Sync latest code
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        copy scripts\standalone.gclient .gclient
+        gclient sync
+    - name: Generate project for update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        gn gen out\Debug --args="webnn_enable_dml=true is_debug=true"
+    - name: Build for update branch
+      shell: cmd
+      run: |
+        set "PATH=%CD%\..\depot_tools;%PATH%"
+        set "DEPOT_TOOLS_WIN_TOOLCHAIN=0"
+        cd update
+        ninja -C out\Debug
+    - name: Check memory leak for update branch
+      shell: cmd
+      run: |
+        cd update
+        echo "Run End2End Tests..."
+        out\Debug\webnn_end2end_tests.exe --gtest_filter=-GemmTests.ScalarBias:Pool2dTests.MaxPool2dDilationsDefault:Pool2dTests.MaxPool2dDilationsNhwc:Pool2dTests.*Pool2dAutoPadExplicitOutputSizes4x4Nhwc:Pool2dTests.*Pool2dAutoPadExplicitRoundingTypeCeilNhwc > update_end2endtests.txt || true
+        python workflow_scripts\memory_leak_check.py update_end2endtests.txt