Merge pull request #708 from Webperf-se/update-version-in-package-json #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Regression Test - Docker Image" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - '**regression-test-docker-image.yml' | |
| - '**regression-test-standard-files.yml' | |
| - '**.py' | |
| - 'Dockerfile' | |
| - '**software-rules.json' | |
| env: | |
| TEST_TAG: webperfse/webperf-core:test | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup QEMU # used to allow multiplatform docker builds | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Setup Docker Buildx # used to allow multiplatform docker builds | |
| uses: docker/setup-buildx-action@v3 | |
| # - name: Login to Docker Hub | |
| # uses: docker/login-action@v3 | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build the Docker image # used to allow multiplatform docker builds | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| load: true | |
| tags: ${{ env.TEST_TAG }} | |
| - name: Check if we have correct python version | |
| run: | | |
| version=$(docker run ${{ env.TEST_TAG }} python --version) | |
| echo "Python version: $version" | |
| if [[ "$version" != "Python 3.13"* ]]; then | |
| echo "Error: Default python version is not 3.13 based, was we unable to change default version?" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core help command works | |
| run: | | |
| help=$(docker run ${{ env.TEST_TAG }} python default.py -h) | |
| echo "$help" | |
| if [[ "$help" != *"-u/--url"* ]]; then | |
| echo "Error: WebPerf-core help command is NOT returning expected content" | |
| exit 1 | |
| fi | |
| - name: Check if dependency tool works | |
| run: | | |
| help=$(docker run ${{ env.TEST_TAG }} python tools/dependency.py) | |
| echo "$help" | |
| if [[ "$help" != *"Python: OK"* ]]; then | |
| echo "Error: dependency tool - Error with Python" | |
| exit 1 | |
| fi | |
| if [[ "$help" != *"Node: OK"* ]]; then | |
| echo "Error: dependency tool - Error with Node" | |
| exit 1 | |
| fi | |
| if [[ "$help" != *"Java: OK"* ]]; then | |
| echo "Error: dependency tool - Error with Java" | |
| exit 1 | |
| fi | |
| if [[ "$help" != *"chrome: OK"* ]]; then | |
| echo "Error: dependency tool - Error with Chrome" | |
| exit 1 | |
| fi | |
| if [[ "$help" != *"firefox: OK"* ]]; then | |
| echo "Error: dependency tool - Error with Firefox" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 1 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 1 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 1 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 1 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| if [[ "$testresult" != *"Performance:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 2 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 2 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 4 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 4 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 5 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 5 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 6 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 6 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 7 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 7 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Standards:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 9 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 9 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 10 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 10 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| if [[ "$testresult" != *"A11y:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 15 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 15 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Performance:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 17 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 17 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 18 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 18 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"A11y:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 20 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 20 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 21 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 21 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 22 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 22 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 22 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Unable to connect to Chrome"* ]]; then | |
| sleep 5 | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 22 -r -i sites.json) | |
| echo "$testresult" | |
| fi | |
| if [[ "$testresult" == *"Error, was unable to load the page you requested"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| if [[ "$testresult" != *"Overall:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 23 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 23 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 24 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 24 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 25 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 25 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"Integrity & Security:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi | |
| - name: Check if webperf_core can run Test 26 | |
| if: always() | |
| run: | | |
| testresult=$(docker run ${{ env.TEST_TAG }} python default.py -t 26 -r -i sites.json) | |
| echo "$testresult" | |
| if [[ "$testresult" != *"A11y:"* ]]; then | |
| echo "Error: WebPerf-core testresult is NOT returning expected review" | |
| exit 1 | |
| fi |