Merge pull request #573 from Webperf-se/is_found-undefined #6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Regression Test - Docker Image" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - '**regression-test-docker-image.yml' | |
| - '**regression-test-standard-files.yml' | |
| - '**.py' | |
| - 'Dockerfile' | |
| - '**software-rules.json' | |
| env: | |
| TEST_TAG: webperfse/webperf-core:test | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup QEMU # used to allow multiplatform docker builds | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Setup Docker Buildx # used to allow multiplatform docker builds | |
| uses: docker/setup-buildx-action@v3 | |
| # - name: Login to Docker Hub | |
| # uses: docker/login-action@v3 | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build the Docker image # used to allow multiplatform docker builds | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| load: true | |
| tags: ${{ env.TEST_TAG }} | |
| - name: Check if we have correct python version | |
| run: | | |
| version=$(docker run ${{ env.TEST_TAG }} python --version) | |
| echo "Python version: $version" | |
| if [[ "$version" != "Python 3.12"* ]]; then | |
| echo "Error: Default python version is not 3.12 based, was we unable to change default version?" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core help command works | |
| run: | | |
| help=$(docker run ${{ env.TEST_TAG }} python default.py -h) | |
| echo "$help" | |
| if [[ "$help" != *"-u/--url"* ]]; then | |
| echo "Error: WebPerf-core help command is NOT returning expected content" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 1 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 1 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 1 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 1 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" != *"Performance:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 2 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 2 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 4 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 5 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 6 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 6 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 7 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 7 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 9 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 9 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 10 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" != *"A11y:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 15 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 15 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Performance:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 17 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 17 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 18 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 18 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"A11y:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 20 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 20 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 21 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 21 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 22 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 22 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 22 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 22 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" != *"Overall:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 23 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 23 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 24 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 24 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 25 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 25 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 26 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 26 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"A11y:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi |