From 1bd2f0a332633966cfe8eec275db83dde3bc679f Mon Sep 17 00:00:00 2001 From: Soren Rasmussen Date: Wed, 23 Oct 2024 16:30:36 -0600 Subject: [PATCH] Adding compare-output github action because the comparison code will be run multiple times for n-core and perfect restart tests --- .github/actions/compare-output/action.yml | 135 ++++++++++++++++++++++ .github/workflows/test-pr.yml | 104 +---------------- 2 files changed, 137 insertions(+), 102 deletions(-) create mode 100644 .github/actions/compare-output/action.yml diff --git a/.github/actions/compare-output/action.yml b/.github/actions/compare-output/action.yml new file mode 100644 index 000000000..1fda14948 --- /dev/null +++ b/.github/actions/compare-output/action.yml @@ -0,0 +1,135 @@ +name: "Compare Output" +description: "Performs a series of comparison steps" +runs: + using: "composite" + steps: + - name: Compare HYDRO_RST.* output with xrcmp + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + for file in output_${{ matrix.configuration }}/HYDRO_RST.*; do\ + python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ + --candidate $file \ + --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ + --log_file $file_diff.txt \ + --n_cores 1; \ + done + + - name: Compare RESTART.* output with xrcmp + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + for file in output_${{ matrix.configuration }}/RESTART.*; do\ + python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ + --candidate $file \ + --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ + --log_file $file_diff.txt \ + --n_cores 1; \ + done + + - name: List file + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) + + - name: Head of list of files + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) + echo "file = ${file}" + + - name: Compare last *.CHANOBS_DOMAIN1 output with xrcmp + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) + python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ + --candidate $file \ + --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ + --log_file $file_diff.txt \ + --n_cores 1 + + - name: Compare last *.CHRTOUT_DOMAIN1 output with xrcmp + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1) + python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ + --candidate $file \ + --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ + --log_file $file_diff.txt \ + --n_cores 1; + + - name: Compare last *.LSMOUT_DOMAIN1 output with xrcmp + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1) + python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ + --candidate $file \ + --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ + --log_file $file_diff.txt \ + --n_cores 1; + + - name: Compare last *.RTOUT_DOMAIN1 output with xrcmp + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1) + python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ + --candidate $file \ + --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ + --log_file $file_diff.txt \ + --n_cores 1; + + - name: Compare output with compare_output + if: ${{ always() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/build/Run + rm -f output_diff + mkdir output_diff + python -c \ + "import sys; \ + sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \ + import compare_output; \ + from pathlib import Path; \ + compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \ + '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \ + '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \ + '${{ matrix.configuration }}')" + + - name: Copy test results from container + if: ${{ always() }} + shell: bash + run: | + mkdir -p $GITHUB_WORKSPACE/test_report + cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/ + + - name: Attach diff plots to PR + if: ${{ failure() }} + shell: bash + run: | + cd $GITHUB_WORKSPACE/candidate/tests/local/utils + bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} + + - name: Archive test results to GitHub + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: test-reports + path: | + ${{ github.workspace }}/test_report/* + + - name: Exit if any tests fail + if: ${{ failure() }} + shell: bash + run: exit 1 diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index 973a5926e..45f0c6187 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -113,105 +113,5 @@ jobs: cd $GITHUB_WORKSPACE/candidate/build/Run make run-croton-${{ matrix.configuration }} - - name: Compare HYDRO_RST.* output with xrcmp - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - for file in output_${{ matrix.configuration }}/HYDRO_RST.*; do\ - python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ - --candidate $file \ - --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ - --log_file $file_diff.txt \ - --n_cores 1; \ - done - - - name: Compare RESTART.* output with xrcmp - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - for file in output_${{ matrix.configuration }}/RESTART.*; do\ - python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ - --candidate $file \ - --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ - --log_file $file_diff.txt \ - --n_cores 1; \ - done - - - name: Compare last *.CHANOBS_DOMAIN1 output with xrcmp - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1) - python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ - --candidate $file \ - --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ - --log_file $file_diff.txt \ - --n_cores 1; \ - - - name: Compare last *.CHRTOUT_DOMAIN1 output with xrcmp - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1) - python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ - --candidate $file \ - --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ - --log_file $file_diff.txt \ - --n_cores 1; \ - - - name: Compare last *.LSMOUT_DOMAIN1 output with xrcmp - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1) - python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ - --candidate $file \ - --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ - --log_file $file_diff.txt \ - --n_cores 1; \ - - - name: Compare last *.RTOUT_DOMAIN1 output with xrcmp - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1) - python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \ - --candidate $file \ - --reference $GITHUB_WORKSPACE/reference/build/Run/$file \ - --log_file $file_diff.txt \ - --n_cores 1; \ - - - name: Compare output with compare_output - if: ${{ always() }} - run: | - cd $GITHUB_WORKSPACE/candidate/build/Run - mkdir output_diff - python -c \ - "import sys; \ - sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \ - import compare_output; \ - from pathlib import Path; \ - compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \ - '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \ - '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \ - '${{ matrix.configuration }}')" - - - name: Copy test results from container - if: ${{ always() }} - run: | - mkdir -p $GITHUB_WORKSPACE/test_report - cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/ - - - name: Attach diff plots to PR - if: ${{ failure() }} - run: | - cd $GITHUB_WORKSPACE/candidate/tests/local/utils - bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} - - - name: Archive test results to GitHub - if: ${{ failure() }} - uses: actions/upload-artifact@v4 - with: - name: test-reports - path: | - ${{ github.workspace }}/test_report/* + - name: Compare output of serial runs + uses: ./github/actions/compare-output