Enabled golden image validation on GitHub CI (#50)

DiligentGraphics · Aug 10, 2022 · 7313e62 · 7313e62
1 parent 881e626
commit 7313e62
Show file tree

Hide file tree

Showing 52 changed files with 47 additions and 38 deletions.
diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
@@ -2,6 +2,12 @@ name: Win32/UWP build
 
 on: [push, pull_request]
 
+env:
+ # 'capture' - generate golden images
+ # 'compare' - render and compare with the references
+ # 'compare_update' - first compare with the references and then update them
+ test_mode: compare_update
+
 jobs:
  precheks:
  name: Win10 -> Pre-checks
@@ -23,13 +29,14 @@ jobs:
  shell: cmd
  working-directory: ${{github.workspace}}/DiligentSamples/BuildTools/FormatValidation
  run: validate_format_win.bat
- 
+
 
  build:
  needs: precheks
  runs-on: windows-2022
 
  strategy:
+ fail-fast: false
  matrix:
  name: ["Win10"]
  toolset: ["Win32", "x64"]
@@ -85,9 +92,15 @@ jobs:
  name: Win10 -> ${{ matrix.name }}-${{ matrix.toolset }}, ${{ matrix.build_type }}
 
  steps:
- - name: Checkout
+ - name: Checkout Code Base
  uses: DiligentGraphics/github-action/checkout@v1
 
+ - name: Checkout Test Data
+ uses: actions/checkout@v2
+ with:
+ repository: DiligentGraphics/DiligentTestData
+ path: DiligentTestData
+
  - name: Set up build environment
  if: success()
  uses: DiligentGraphics/github-action/setup-build-env@v1
@@ -106,5 +119,33 @@ jobs:
  cmake-args: ${{ matrix.cmake_args }}
 
  - name: Build
+ id: build
  if: success()
  uses: DiligentGraphics/github-action/build@v1
+
+ - name: Run Sample Tests D3D11
+ working-directory: ${{github.workspace}}/DiligentSamples/Tests
+ if: ${{ steps.build.outcome == 'success' && (matrix.name == 'Win10' || matrix.name == 'Win8.1') }}
+ shell: cmd
+ run: |
+ :: Run Sample Tests D3D11
+ set golden_images_root_dir=${{github.workspace}}/DiligentTestData/GoldenImages
+ ProcessGoldenImages.bat ${{env.DILIGENT_BUILD_DIR}} ${{matrix.build_type}} ${{env.test_mode}} "--mode d3d11 --adapter sw"
+
+ - name: Run Sample Tests D3D12
+ working-directory: ${{github.workspace}}/DiligentSamples/Tests
+ if: ${{ steps.build.outcome == 'success' && matrix.name == 'Win10' }}
+ shell: cmd
+ run: |
+ :: Run Sample Tests D3D12
+ set golden_images_root_dir=${{github.workspace}}/DiligentTestData/GoldenImages
+ ProcessGoldenImages.bat ${{env.DILIGENT_BUILD_DIR}} ${{matrix.build_type}} ${{env.test_mode}} "--mode d3d12 --adapter sw"
+
+ - name: Upload Golden Images
+ uses: actions/upload-artifact@v2
+ if: ${{ steps.build.outcome == 'success' && (matrix.name == 'Win10' || matrix.name == 'Win8.1') }}
+ with:
+ name: GoldenImages-${{ matrix.name }}-${{ matrix.toolset }}-${{ matrix.build_type }}
+ path: |
+ ${{github.workspace}}/DiligentTestData/GoldenImages
+ retention-days: 90
diff --git a/BuildTools/Scripts/appveyor/run_tests.bat b/BuildTools/Scripts/appveyor/run_tests.bat
diff --git a/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d11.png b/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d12.png b/Tests/GoldenImages/Samples/Atmosphere/Atmosphere_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d11.png b/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d12.png b/Tests/GoldenImages/Samples/GLTFViewer/GLTFViewer_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d11.png b/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d12.png b/Tests/GoldenImages/Samples/ImguiDemo/ImguiDemo_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d11.png b/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d12.png b/Tests/GoldenImages/Samples/NuklearDemo/NuklearDemo_gi_d3d12.png
diff --git a/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d11.png b/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d11.png
diff --git a/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d12.png b/Tests/GoldenImages/Samples/Shadows/Shadows_gi_d3d12.png
diff --git a/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d11.png b/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d11.png
diff --git a/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d12.png b/...Images/Tutorials/Tutorial01_HelloTriangle/Tutorial01_HelloTriangle_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial02_Cube/Tutorial02_Cube_gi_d3d12.png
diff --git a/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d11.png b/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d11.png
diff --git a/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d12.png b/...ldenImages/Tutorials/Tutorial03_Texturing-C/Tutorial03_Texturing-C_gi_d3d12.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d11.png b/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d11.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d12.png b/...s/GoldenImages/Tutorials/Tutorial03_Texturing/Tutorial03_Texturing_gi_d3d12.png
diff --git a/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d11.png b/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d11.png
diff --git a/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d12.png b/...GoldenImages/Tutorials/Tutorial04_Instancing/Tutorial04_Instancing_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d11.png b/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d12.png b/...enImages/Tutorials/Tutorial05_TextureArray/Tutorial05_TextureArray_gi_d3d12.png
diff --git a/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d11.png b/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d11.png
diff --git a/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d12.png b/...ages/Tutorials/Tutorial06_Multithreading/Tutorial06_Multithreading_gi_d3d12.png
diff --git a/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d11.png b/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d11.png
diff --git a/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d12.png b/...ages/Tutorials/Tutorial07_GeometryShader/Tutorial07_GeometryShader_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d11.png b/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d12.png b/...enImages/Tutorials/Tutorial08_Tessellation/Tutorial08_Tessellation_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial09_Quads/Tutorial09_Quads_gi_d3d12.png
diff --git a/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d11.png b/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d11.png
diff --git a/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d12.png b/...Images/Tutorials/Tutorial10_DataStreaming/Tutorial10_DataStreaming_gi_d3d12.png
diff --git a/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d11.png b/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d11.png
diff --git a/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d12.png b/...es/Tutorials/Tutorial11_ResourceUpdates/Tutorial11_ResourceUpdates_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d11.png b/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d12.png b/...enImages/Tutorials/Tutorial12_RenderTarget/Tutorial12_RenderTarget_gi_d3d12.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d11.png b/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d11.png
diff --git a/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d12.png b/...s/GoldenImages/Tutorials/Tutorial13_ShadowMap/Tutorial13_ShadowMap_gi_d3d12.png
diff --git a/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d11.png b/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d11.png
diff --git a/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d12.png b/...Images/Tutorials/Tutorial14_ComputeShader/Tutorial14_ComputeShader_gi_d3d12.png
diff --git a/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d11.png b/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d11.png
diff --git a/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d12.png b/...utorials/Tutorial16_BindlessResources/Tutorial16_BindlessResources_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial17_MSAA/Tutorial17_MSAA_gi_d3d12.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d11.png b/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d11.png
diff --git a/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d12.png b/Tests/GoldenImages/Tutorials/Tutorial18_Queries/Tutorial18_Queries_gi_d3d12.png
diff --git a/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d11.png b/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d11.png
diff --git a/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d12.png b/...enImages/Tutorials/Tutorial19_RenderPasses/Tutorial19_RenderPasses_gi_d3d12.png
diff --git a/Tests/GoldenImages/ProcessGoldenImages.bat → Tests/ProcessGoldenImages.bat b/Tests/GoldenImages/ProcessGoldenImages.bat → Tests/ProcessGoldenImages.bat
@@ -29,7 +29,7 @@ if "%golden_img_height%" == "" (
  set golden_img_height=512
 )
 if "%golden_images_root_dir%" == "" (
- set golden_images_root_dir=../../../Tests/GoldenImages
+ set golden_images_root_dir=.
 )
 
  :: ~ removes surrounding quotes
@@ -51,7 +51,7 @@ set rest_args=
  goto loop1
 :end_loop1
 
-cd ../..
+cd ..
 
 set Tutorials=Tutorial01_HelloTriangle^
  Tutorial02_Cube^
@@ -93,7 +93,7 @@ for %%X in (%Samples%) do (
  set /a APP_ID=!APP_ID!*2
 )
 
-cd Tests/GoldenImages
+cd Tests
 
 EXIT /B %ERROR%
 
@@ -155,7 +155,7 @@ EXIT /B %ERROR%
  )
  )
  set capture_name=%app_name%_gi_!backend_name!
- 
+
 
  rem !!! ERRORLEVEL doesn't get updated inside control blocks like IF statements unless !!!
  rem !!! !ERRORLEVEL! is used instead of %ERRORLEVEL% and delayed expansion is enabled as below: !!!

diff --git a/appveyor.yml b/appveyor.yml
@@ -10,11 +10,6 @@ image:
 - Visual Studio 2019
 
 environment:
- # use 'capture' to generate golden images and upload them as build artifacts
- # use 'compare' to render and compare with the golden references
- # use 'compare_update' to first compare with the references and then update them
- test_mode: compare_update
-
  matrix:
  - platform: x64
  configuration: Debug
@@ -52,15 +47,3 @@ build:
  project: c:\projects\build\DiligentSamples_Test.sln
  verbosity: minimal
  parallel: true
-
-test_script:
- - 'cd %APPVEYOR_BUILD_FOLDER%\Tests\GoldenImages'
- - '%APPVEYOR_BUILD_FOLDER%\BuildTools\Scripts\appveyor\run_tests.bat c:\projects\build %test_mode%'
-
-after_test:
- - 'cd %APPVEYOR_BUILD_FOLDER%'
- - 7z a golden_images-"%CONFIGURATION%".zip .\Tests\GoldenImages\*\*
-
-artifacts:
- - path: \golden_images-*.zip
- name: golden_images-zip