Merge pull request #1974 from ellraiser/12.0-testsuite

basic API test suite

.github/workflows/main.yml

@@ -3,7 +3,13 @@ on: [push, pull_request]
 
 jobs:
  linux-os:
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
+ permissions:
+ checks: write
+ pull-requests: write
+ env:
+ ALSOFT_CONF: love2d-${{ github.sha }}/testing/resources/alsoft.conf
+ DISPLAY: :99
  steps:
  - name: Update APT
  run: sudo apt-get update
@@ -16,7 +22,8 @@ jobs:
  libgl1-mesa-dev libdbus-1-dev libudev-dev libgles2-mesa-dev \
  libegl1-mesa-dev libibus-1.0-dev fcitx-libs-dev libsamplerate0-dev \
  libsndio-dev libwayland-dev libxkbcommon-dev libdrm-dev libgbm-dev \
- libcurl4-openssl-dev
+ libcurl4-openssl-dev libfuse2 wmctrl openbox mesa-vulkan-drivers \
+ libvulkan1 vulkan-tools vulkan-validationlayers
  - name: Checkout love-appimage-source
  uses: actions/checkout@v3
  with:
@@ -40,6 +47,83 @@ jobs:
  run: make LOVE_BRANCH=${{ github.sha }}
  - name: Print LuaJIT branch
  run: git -C LuaJIT-v2.1 branch -v
+ # start xvfb for test running
+ - name: Start xvfb and openbox
+ run: |
+ echo "Starting XVFB on $DISPLAY"
+ Xvfb $DISPLAY -screen 0, 360x240x24 &
+ echo "XVFBPID=$!" >> $GITHUB_ENV
+ # wait for xvfb to startup (3s is the same amount xvfb-run waits by default)
+ sleep 3
+ openbox &
+ echo "OPENBOXPID=$!" >> $GITHUB_ENV
+ # linux opengl tests
+ - name: Run Test Suite (opengl)
+ run: |
+ chmod a+x love-${{ github.sha }}.AppImage
+ ./love-${{ github.sha }}.AppImage love2d-${{ github.sha }}/testing/main.lua --runAllTests --isRunner
+ - name: Love Test Report (opengl)
+ id: report1
+ uses: ellraiser/love-test-report@main
+ with:
+ name: Love Testsuite Linux
+ title: test-report-linux-opengl
+ path: love2d-${{ github.sha }}/testing/output/lovetest_runAllTests.md
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Zip Test Output (opengl)
+ run: |
+ 7z a -tzip test-output-linux-opengl.zip love2d-${{ github.sha }}/testing/output/
+ - name: Artifact Test Output (opengl)
+ uses: actions/upload-artifact@v3
+ with:
+ name: test-output-linux-opengl-${{ steps.report1.outputs.conclusion }}
+ path: test-output-linux-opengl.zip
+ # linux opengles tests
+ - name: Run Test Suite (opengles)
+ run: |
+ export LOVE_GRAPHICS_USE_OPENGLES=1
+ ./love-${{ github.sha }}.AppImage love2d-${{ github.sha }}/testing/main.lua --runAllTests --isRunner
+ - name: Love Test Report (opengles)
+ uses: ellraiser/love-test-report@main
+ id: report2
+ with:
+ name: Love Testsuite Linux
+ title: test-report-linux-opengles
+ path: love2d-${{ github.sha }}/testing/output/lovetest_runAllTests.md
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Zip Test Output (opengles)
+ run: |
+ 7z a -tzip test-output-linux-opengles.zip love2d-${{ github.sha }}/testing/output/
+ - name: Artifact Test Output (opengles)
+ uses: actions/upload-artifact@v3
+ with:
+ name: test-output-linux-opengles-${{ steps.report2.outputs.conclusion }}
+ path: test-output-linux-opengles.zip
+# # linux vulkan tests
+# - name: Run Test Suite (vulkan)
+# run: |
+# export LOVE_GRAPHICS_DEBUG=1
+# ./love-${{ github.sha }}.AppImage love2d-${{ github.sha }}/testing/main.lua --runAllTests --isRunner --renderers vulkan
+# - name: Love Test Report (vulkan)
+# uses: ellraiser/love-test-report@main
+# with:
+# name: Love Testsuite Linux
+# title: test-report-linux-vulkan
+# path: love2d-${{ github.sha }}/testing/output/lovetest_runAllTests.md
+# - name: Zip Test Output (vulkan)
+# run: |
+# 7z a -tzip test-output-linux-vulkan.zip love2d-${{ github.sha }}/testing/output/
+# - name: Artifact Test Output (vulkan)
+# uses: actions/upload-artifact@v3
+# with:
+# name: test-output-linux-vulkan
+# path: test-output-linux-vulkan.zip
+ - name: Stop xvfb and openbox
+ # should always stop xvfb and openbox even if other steps failed
+ if: always()
+ run: |
+ kill $XVFBPID
+ kill $OPENBOXPID
  - name: Artifact
  uses: actions/upload-artifact@v3
  with:
@@ -50,8 +134,21 @@ jobs:
  with:
  name: love-x86_64-AppImage-debug
  path: love-${{ github.sha }}.AppImage-debug.tar.gz
+ - name: Check Tests Passing
+ if: steps.report1.outputs.conclusion == 'failure' || steps.report2.outputs.conclusion == 'failure'
+ run: |
+ echo "${{ steps.report1.outputs.failed }} opengl tests failed"
+ echo "${{ steps.report2.outputs.failed }} opengles tests failed"
+ exit 1
  windows-os:
  runs-on: windows-latest
+ permissions:
+ checks: write
+ pull-requests: write
+ env: 
+ ALSOFT_CONF: megasource/libs/love/testing/resources/alsoft.conf
+ VK_ICD_FILENAMES: ${{ github.workspace }}\mesa\x64\lvp_icd.x86_64.json
+ VULKAN_SDK: C:/VulkanSDK/1.3.231.1
  strategy:
  matrix:
  platform: [Win32, x64, ARM64]
@@ -206,8 +303,115 @@ jobs:
  with:
  name: love-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-dbg
  path: pdb/Release/*.pdb
+ # install mesa for graphic tests
+ - name: Install Mesa 
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ run: |
+ curl -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/23.2.1/mesa3d-23.2.1-release-msvc.7z
+ 7z x mesa.7z -o*
+ powershell.exe mesa\systemwidedeploy.cmd 1
+ # build love to use for the tests
+ - name: Build Test Exe
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ run: cmake --build build --config Release --target install
+ # windows opengl tests
+ - name: Run Tests (opengl)
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ run: |
+ echo 'check dir'
+ ls
+ powershell.exe ./install/lovec.exe ./megasource/libs/love/testing/main.lua --runAllTests --isRunner
+ - name: Love Test Report (opengl)
+ id: report1
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ uses: ellraiser/love-test-report@main
+ with:
+ name: Love Testsuite Windows ${{ steps.vars.outputs.arch }} ${{ steps.vars.outputs.compatname }} (opengl)
+ title: test-report-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengl
+ path: megasource/libs/love/testing/output/lovetest_runAllTests.md
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Zip Test Output (opengl)
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ run: |
+ 7z a -tzip test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengl.zip megasource/libs/love/testing/output/
+ - name: Artifact Test Output (opengl)
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ uses: actions/upload-artifact@v3
+ with:
+ name: test-output-windows-${{ steps.vars.outputs.arch }}-opengl-${{ steps.report1.outputs.conclusion }}
+ path: test-output-windows-${{ steps.vars.outputs.arch }}-opengl.zip
+ # windows opengles tests
+ - name: Run Tests (opengles)
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ run: |
+ $ENV:LOVE_GRAPHICS_USE_OPENGLES=1
+ powershell.exe ./install/lovec.exe ./megasource/libs/love/testing/main.lua --runAllTests --isRunner
+ - name: Love Test Report (opengles)
+ id: report2
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ uses: ellraiser/love-test-report@main
+ with:
+ name: Love Testsuite Windows ${{ steps.vars.outputs.arch }} ${{ steps.vars.outputs.compatname }} (opengles)
+ title: test-report-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengles
+ path: megasource/libs/love/testing/output/lovetest_runAllTests.md
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Zip Test Output (opengles)
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ run: |
+ 7z a -tzip test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengles.zip megasource/libs/love/testing/output/
+ - name: Artifact Test Output (opengles)
+ if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat'
+ uses: actions/upload-artifact@v3
+ with:
+ name: test-output-windows-${{ steps.vars.outputs.arch }}-opengles-${{ steps.report2.outputs.conclusion }}
+ path: test-output-windows-${{ steps.vars.outputs.arch }}-opengles.zip
+ - name: Check Tests Passing
+ if: steps.report1.outputs.conclusion == 'failure' || steps.report2.outputs.conclusion == 'failure'
+ run: |
+ echo "${{ steps.report1.outputs.failed }} opengl tests failed"
+ echo "${{ steps.report2.outputs.failed }} opengles tests failed"
+ exit 1
+# # install vulkan
+# - name: Install Vulkan
+# if: steps.vars.outputs.arch != 'ARM64'
+# run: |
+# curl -L --show-error --output VulkanSDK.exe https://sdk.lunarg.com/sdk/download/1.3.231.1/windows/VulkanSDK-1.3.231.1-Installer.exe
+# ./VulkanSDK.exe --root C:/VulkanSDK/1.3.231.1 --accept-licenses --default-answer --confirm-command install com.lunarg.vulkan.core com.lunarg.vulkan.vma 
+# curl -L --show-error --output vulkan-runtime.zip https://sdk.lunarg.com/sdk/download/1.3.231.1/windows/vulkan-runtime-components.zip
+# 7z e vulkan-runtime.zip -o"C:/VulkanSDK/1.3.231.1/runtime/x64" */x64
+# copy "C:/VulkanSDK/1.3.231.1/runtime/x64/vulkan-1.dll" "mesa/x64"
+# copy "C:/VulkanSDK/1.3.231.1/runtime/x64/vulkan-1.dll" "C:/Windows/System32"
+# copy "C:/VulkanSDK/1.3.231.1/runtime/x64/vulkan-1.dll" "love-12.0-win64/love-12.0-win64"
+# reg add HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\Vulkan\Drivers /v "${{ github.workspace }}\mesa\x64\lvp_icd.x86_64.json" /t REG_DWORD /d 0
+# powershell.exe C:/VulkanSDK/1.3.231.1/runtime/x64/vulkaninfo.exe --summary
+# # windows vulkan tests
+# - name: Run Tests (vulkan)
+# if: steps.vars.outputs.arch != 'ARM64'
+# run: |
+# $ENV:LOVE_GRAPHICS_DEBUG=1
+# powershell.exe ./install/lovec.exe ./megasource/libs/love/testing/main.lua --runAllTests --isRunner --renderers vulkan
+# - name: Love Test Report (vulkan)
+# if: steps.vars.outputs.arch != 'ARM64'
+# uses: ellraiser/love-test-report@main
+# with:
+# name: Love Testsuite Windows ${{ steps.vars.outputs.arch }} ${{ steps.vars.outputs.compatname }} (vulkan)
+# title: test-report-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan
+# path: megasource/libs/love/testing/output/lovetest_runAllTests.md
+# - name: Zip Test Output (vulkan)
+# if: steps.vars.outputs.arch != 'ARM64'
+# run: |
+# 7z a -tzip test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan.zip megasource/libs/love/testing/output/
+# - name: Artifact Test Output (vulkan)
+# if: steps.vars.outputs.arch != 'ARM64'
+# uses: actions/upload-artifact@v3
+# with:
+# name: test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan
+# path: test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan.zip
  macOS:
  runs-on: macos-latest
+ permissions:
+ checks: write
+ pull-requests: write
  steps:
  - name: Checkout
  uses: actions/checkout@v3
@@ -234,6 +438,32 @@ jobs:
  with:
  name: love-macos
  path: love-macos.zip
+ # macos opengl tests (metal not supported on runners)
+ - name: Run Test Suite
+ run: |
+ ls
+ love-macos/love.app/Contents/MacOS/love ./testing/main.lua --runAllTests --isRunner
+ - name: Love Test Report
+ id: report1
+ uses: ellraiser/love-test-report@main
+ with:
+ name: Love Testsuite MacOS
+ title: test-report-macos
+ path: testing/output/lovetest_runAllTests.md
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Zip Test Output
+ run: |
+ 7z a -tzip test-output-macos-opengl.zip ./testing/output/
+ - name: Artifact Test Output
+ uses: actions/upload-artifact@v3
+ with:
+ name: test-output-macos-opengl-${{ steps.report1.outputs.conclusion }}
+ path: test-output-macos-opengl.zip
+ - name: Check Tests Passing
+ if: steps.report1.outputs.conclusion == 'failure'
+ run: |
+ echo "${{ steps.report1.outputs.failed }} opengl tests failed"
+ exit 1
  iOS-Simulator:
  runs-on: macos-latest
  steps:

.gitignore

@@ -68,4 +68,8 @@ stamp-h1
 /src/love
 /src/tags
 .vs/
-.vscode/
+.vscode/
+/testing/output/*.xml
+/testing/output/*.html
+/testing/output/*.md
+/testing/output/actual/*.png