From c8f7fa3d1fe7255378928c041cce0298fa9ef546 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Fri, 10 Nov 2023 13:31:49 +0100 Subject: [PATCH 01/14] Add fork repositories support for regression tests report --- .github/workflows/main.yml | 28 ++++++++++++++---- .github/workflows/tests_results.yml | 44 +++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/tests_results.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 957e2b1c5..1ee09d0d2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,16 @@ on: workflow_dispatch: jobs: + event_file: + name: Event file + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: Event File + path: ${{ github.event_path }} + build-regression-tests: name: Build regression tests runs-on: ubuntu-latest @@ -68,12 +78,18 @@ jobs: . venv/bin/activate scripts/run_tests.py -a regression - - name: Test Report - uses: EnricoMi/publish-unit-test-result-action@v2 + # - name: Test Report + # uses: EnricoMi/publish-unit-test-result-action@v2 + # with: + # files: test/regression/cocotb/results.xml + # check_name: cocotb test results + # comment_mode: off + + - name: Upload tests results + uses: actions/upload-artifact@v3 with: - files: test/regression/cocotb/results.xml - check_name: cocotb test results - comment_mode: off + name: Tests results + path: test/regression/cocotb/results.xml unit-test: name: Run unit tests @@ -102,7 +118,7 @@ jobs: run: ./scripts/run_tests.py --verbose - name: Check traces - run: ./scripts/run_tests.py -t -c 1 TestCore + run: ./scripts/run_tests.py -t -c 1 TestCore lint: name: Check code formatting and typing diff --git a/.github/workflows/tests_results.yml b/.github/workflows/tests_results.yml new file mode 100644 index 000000000..9f375c192 --- /dev/null +++ b/.github/workflows/tests_results.yml @@ -0,0 +1,44 @@ +name: Tests results + +on: + workflow_run: + workflows: ["CI"] + types: + - completed +permissions: {} + +jobs: + test-results: + name: Tests results + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + + permissions: + checks: write + + # needed unless run with comment_mode: off + pull-requests: write + + # only needed for private repository + contents: read + + # only needed for private repository + issues: read + + # required by download step to access artifacts API + actions: read + + steps: + - name: Download and Extract Artifacts + uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 + with: + run_id: ${{ github.event.workflow_run.id }} + path: artifacts + + - name: Publish tests results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + files: test/regression/cocotb/results.xml From a1572516d73b19bdc4e0753bcf8f46fa4edc059a Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 12 Nov 2023 22:36:12 +0100 Subject: [PATCH 02/14] Adjust changes in workflow and apply fixes --- .github/workflows/main.yml | 24 ++++------------- .../{tests_results.yml => test_results.yml} | 27 +++++++------------ 2 files changed, 14 insertions(+), 37 deletions(-) rename .github/workflows/{tests_results.yml => test_results.yml} (55%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ee09d0d2..1d2f9ed29 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,15 +12,6 @@ on: workflow_dispatch: jobs: - event_file: - name: Event file - runs-on: ubuntu-latest - steps: - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: Event File - path: ${{ github.event_path }} build-regression-tests: name: Build regression tests @@ -78,18 +69,13 @@ jobs: . venv/bin/activate scripts/run_tests.py -a regression - # - name: Test Report - # uses: EnricoMi/publish-unit-test-result-action@v2 - # with: - # files: test/regression/cocotb/results.xml - # check_name: cocotb test results - # comment_mode: off - - - name: Upload tests results + - name: Upload artifacts uses: actions/upload-artifact@v3 with: - name: Tests results - path: test/regression/cocotb/results.xml + name: "regression_tests_artifacts" + path: | + test/regression/cocotb/results.xml + ${{ github.event_path }} unit-test: name: Run unit tests diff --git a/.github/workflows/tests_results.yml b/.github/workflows/test_results.yml similarity index 55% rename from .github/workflows/tests_results.yml rename to .github/workflows/test_results.yml index 9f375c192..2f4b95d1d 100644 --- a/.github/workflows/tests_results.yml +++ b/.github/workflows/test_results.yml @@ -1,4 +1,4 @@ -name: Tests results +name: Test results on: workflow_run: @@ -8,37 +8,28 @@ on: permissions: {} jobs: - test-results: - name: Tests results + make-test-report: + name: Make test report runs-on: ubuntu-latest if: github.event.workflow_run.conclusion != 'skipped' permissions: checks: write - - # needed unless run with comment_mode: off - pull-requests: write - - # only needed for private repository - contents: read - - # only needed for private repository - issues: read - - # required by download step to access artifacts API actions: read steps: - - name: Download and Extract Artifacts + - name: Download and extract artifacts uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 with: run_id: ${{ github.event.workflow_run.id }} path: artifacts - - name: Publish tests results + - name: Publish test results uses: EnricoMi/publish-unit-test-result-action@v2 with: + check_name: cocotb test results + comment_mode: off commit: ${{ github.event.workflow_run.head_sha }} - event_file: artifacts/Event File/event.json + event_file: artifacts/regression_tests_artifacts/event.json event_name: ${{ github.event.workflow_run.event }} - files: test/regression/cocotb/results.xml + files: artifacts/regression_tests_artifacts/test/regression/cocotb/results.xml From 3995e43a0572dd99c4b4ed78189cfb2f6e1fa6a5 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 12 Nov 2023 22:45:18 +0100 Subject: [PATCH 03/14] Fix dawidd6/action-download-artifact tag --- .github/workflows/test_results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_results.yml b/.github/workflows/test_results.yml index 2f4b95d1d..20982ef93 100644 --- a/.github/workflows/test_results.yml +++ b/.github/workflows/test_results.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Download and extract artifacts - uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 + uses: dawidd6/action-download-artifact@v2 with: run_id: ${{ github.event.workflow_run.id }} path: artifacts From 6ed77c1ebf1c6d0433fce7b1ba3e621d52ca2200 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 12 Nov 2023 22:48:39 +0100 Subject: [PATCH 04/14] Fix indentations --- .github/workflows/test_results.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_results.yml b/.github/workflows/test_results.yml index 20982ef93..fc90a08c1 100644 --- a/.github/workflows/test_results.yml +++ b/.github/workflows/test_results.yml @@ -19,10 +19,10 @@ jobs: steps: - name: Download and extract artifacts - uses: dawidd6/action-download-artifact@v2 - with: - run_id: ${{ github.event.workflow_run.id }} - path: artifacts + uses: dawidd6/action-download-artifact@v2 + with: + run_id: ${{ github.event.workflow_run.id }} + path: artifacts - name: Publish test results uses: EnricoMi/publish-unit-test-result-action@v2 From 1b033dfb2117c68984cddd850a04e59052b4c7e5 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 12 Nov 2023 22:56:18 +0100 Subject: [PATCH 05/14] Add param to download only specific artifacts --- .github/workflows/test_results.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_results.yml b/.github/workflows/test_results.yml index fc90a08c1..3d6d201b3 100644 --- a/.github/workflows/test_results.yml +++ b/.github/workflows/test_results.yml @@ -14,15 +14,16 @@ jobs: if: github.event.workflow_run.conclusion != 'skipped' permissions: - checks: write actions: read + checks: write steps: - name: Download and extract artifacts uses: dawidd6/action-download-artifact@v2 with: - run_id: ${{ github.event.workflow_run.id }} + name: regression_tests_artifacts path: artifacts + run_id: ${{ github.event.workflow_run.id }} - name: Publish test results uses: EnricoMi/publish-unit-test-result-action@v2 From a7af34c75349aa42f941fda48436e954eb891222 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 12 Nov 2023 23:29:04 +0100 Subject: [PATCH 06/14] Upload and download artifacts separately --- .github/workflows/main.yml | 10 ++++++++-- .github/workflows/test_results.yml | 15 +++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1d2f9ed29..6ec6bc224 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,12 +69,18 @@ jobs: . venv/bin/activate scripts/run_tests.py -a regression - - name: Upload artifacts + - name: Upload test results uses: actions/upload-artifact@v3 with: - name: "regression_tests_artifacts" + name: "regression_tests_results" path: | test/regression/cocotb/results.xml + + - name: Upload event file + uses: actions/upload-artifact@v3 + with: + name: "event_file" + path: | ${{ github.event_path }} unit-test: diff --git a/.github/workflows/test_results.yml b/.github/workflows/test_results.yml index 3d6d201b3..3902cf864 100644 --- a/.github/workflows/test_results.yml +++ b/.github/workflows/test_results.yml @@ -18,10 +18,17 @@ jobs: checks: write steps: - - name: Download and extract artifacts + - name: Download test results uses: dawidd6/action-download-artifact@v2 with: - name: regression_tests_artifacts + name: regression_tests_results + path: artifacts + run_id: ${{ github.event.workflow_run.id }} + + - name: Download event file + uses: dawidd6/action-download-artifact@v2 + with: + name: event_file path: artifacts run_id: ${{ github.event.workflow_run.id }} @@ -31,6 +38,6 @@ jobs: check_name: cocotb test results comment_mode: off commit: ${{ github.event.workflow_run.head_sha }} - event_file: artifacts/regression_tests_artifacts/event.json + event_file: artifacts/event_file/event.json event_name: ${{ github.event.workflow_run.event }} - files: artifacts/regression_tests_artifacts/test/regression/cocotb/results.xml + files: artifacts/regression_tests_results/results.xml From 9fe1d26d80522c4acdacd68238e0883ae9b2407e Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 01:54:57 +0100 Subject: [PATCH 07/14] Change the way of checking regression test results --- .github/workflows/main.yml | 16 ++--------- .github/workflows/test_results.yml | 43 ------------------------------ scripts/check_test_results.py | 17 ++++++++++++ 3 files changed, 19 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/test_results.yml create mode 100755 scripts/check_test_results.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ec6bc224..909e947f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,6 @@ on: workflow_dispatch: jobs: - build-regression-tests: name: Build regression tests runs-on: ubuntu-latest @@ -69,19 +68,8 @@ jobs: . venv/bin/activate scripts/run_tests.py -a regression - - name: Upload test results - uses: actions/upload-artifact@v3 - with: - name: "regression_tests_results" - path: | - test/regression/cocotb/results.xml - - - name: Upload event file - uses: actions/upload-artifact@v3 - with: - name: "event_file" - path: | - ${{ github.event_path }} + - name: Check regression occurrence + run: ./scripts/check_test_results.py unit-test: name: Run unit tests diff --git a/.github/workflows/test_results.yml b/.github/workflows/test_results.yml deleted file mode 100644 index 3902cf864..000000000 --- a/.github/workflows/test_results.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Test results - -on: - workflow_run: - workflows: ["CI"] - types: - - completed -permissions: {} - -jobs: - make-test-report: - name: Make test report - runs-on: ubuntu-latest - if: github.event.workflow_run.conclusion != 'skipped' - - permissions: - actions: read - checks: write - - steps: - - name: Download test results - uses: dawidd6/action-download-artifact@v2 - with: - name: regression_tests_results - path: artifacts - run_id: ${{ github.event.workflow_run.id }} - - - name: Download event file - uses: dawidd6/action-download-artifact@v2 - with: - name: event_file - path: artifacts - run_id: ${{ github.event.workflow_run.id }} - - - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - check_name: cocotb test results - comment_mode: off - commit: ${{ github.event.workflow_run.head_sha }} - event_file: artifacts/event_file/event.json - event_name: ${{ github.event.workflow_run.event }} - files: artifacts/regression_tests_results/results.xml diff --git a/scripts/check_test_results.py b/scripts/check_test_results.py new file mode 100755 index 000000000..c1316e1e5 --- /dev/null +++ b/scripts/check_test_results.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import sys +import os +import pathlib +import xml.etree.ElementTree as ET + +FAILURE_TAG = 'failure' +TOP_DIR = pathlib.Path(__file__).parent.parent +TEST_RESULTS_FILE = TOP_DIR.joinpath('test/regression/cocotb/results.xml') + +if not os.path.exists(TEST_RESULTS_FILE): + sys.exit(1) + +tree = ET.parse(TEST_RESULTS_FILE) +if len(tree.findall(FAILURE_TAG)) > 0: + sys.exit(1) From 5a007d7ea90bfcf187ff7f31746a50b6fa861166 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 02:10:17 +0100 Subject: [PATCH 08/14] Fix formatting --- scripts/check_test_results.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/check_test_results.py b/scripts/check_test_results.py index c1316e1e5..728b84c3f 100755 --- a/scripts/check_test_results.py +++ b/scripts/check_test_results.py @@ -3,15 +3,15 @@ import sys import os import pathlib -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as eT -FAILURE_TAG = 'failure' +FAILURE_TAG = "failure" TOP_DIR = pathlib.Path(__file__).parent.parent -TEST_RESULTS_FILE = TOP_DIR.joinpath('test/regression/cocotb/results.xml') +TEST_RESULTS_FILE = TOP_DIR.joinpath("test/regression/cocotb/results.xml") if not os.path.exists(TEST_RESULTS_FILE): sys.exit(1) -tree = ET.parse(TEST_RESULTS_FILE) +tree = eT.parse(TEST_RESULTS_FILE) if len(tree.findall(FAILURE_TAG)) > 0: sys.exit(1) From eb1eebfbd53545aff6f761b62dddae5d22b80b91 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 02:24:52 +0100 Subject: [PATCH 09/14] Test failing if workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 909e947f0..2dcb1e584 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: - name: Generate Verilog run: | . venv/bin/activate - PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config full + PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config tiny - uses: actions/download-artifact@v3 with: From eef936e518f8e71bc46e0e2287dae8f55e3cb7c2 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 02:44:56 +0100 Subject: [PATCH 10/14] Add prints --- scripts/check_test_results.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/check_test_results.py b/scripts/check_test_results.py index 728b84c3f..e72fc8135 100755 --- a/scripts/check_test_results.py +++ b/scripts/check_test_results.py @@ -10,8 +10,13 @@ TEST_RESULTS_FILE = TOP_DIR.joinpath("test/regression/cocotb/results.xml") if not os.path.exists(TEST_RESULTS_FILE): + print("File not found: ", TEST_RESULTS_FILE) sys.exit(1) tree = eT.parse(TEST_RESULTS_FILE) + if len(tree.findall(FAILURE_TAG)) > 0: + print("Some regression tests failed") sys.exit(1) + +print("All regression tests pass") From 15e229c69e3c7fe21df975ffa1663529f99cf799 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 02:50:20 +0100 Subject: [PATCH 11/14] Test failing workflow - second try --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2dcb1e584..ad82843ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: - name: Generate Verilog run: | . venv/bin/activate - PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config tiny + PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config basic - uses: actions/download-artifact@v3 with: From cc4b51c578aa613045764592b93f3faa7e9582b3 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 03:23:57 +0100 Subject: [PATCH 12/14] Fix bug --- scripts/check_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_test_results.py b/scripts/check_test_results.py index e72fc8135..c10af9bc2 100755 --- a/scripts/check_test_results.py +++ b/scripts/check_test_results.py @@ -15,7 +15,7 @@ tree = eT.parse(TEST_RESULTS_FILE) -if len(tree.findall(FAILURE_TAG)) > 0: +if len(list(tree.iter(FAILURE_TAG))) > 0: print("Some regression tests failed") sys.exit(1) From 8c00b716d3fc7236081d16c3ecaba05ce6e80a29 Mon Sep 17 00:00:00 2001 From: Maksymilian Komarnicki Date: Sun, 19 Nov 2023 03:25:27 +0100 Subject: [PATCH 13/14] Revert test failing workflow changes --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ad82843ed..909e947f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: - name: Generate Verilog run: | . venv/bin/activate - PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config basic + PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config full - uses: actions/download-artifact@v3 with: From 2d3c7613fb5159cdb7e8b0c30af86b54f8ea9ccf Mon Sep 17 00:00:00 2001 From: Marek Materzok Date: Mon, 27 Nov 2023 10:16:31 +0100 Subject: [PATCH 14/14] Update .github/workflows/main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 909e947f0..78a23656a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,7 +68,7 @@ jobs: . venv/bin/activate scripts/run_tests.py -a regression - - name: Check regression occurrence + - name: Check for test failure run: ./scripts/check_test_results.py unit-test: