Skip to content

Commit

Permalink
Use lower bound for expected findings in self-test-and-release.yml (#32)
Browse files Browse the repository at this point in the history
This check breaks frequently when new rules get added or old rules get refined because the exact number of recommendations in these packages changes. Instead, use a lower bound. This still tests if the CLI returned a non-trivial number of recommendations, but it becomes less flaky when rules change.
  • Loading branch information
martinschaef authored May 22, 2023
1 parent 8957ced commit 2bc43bc
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/self-test-and-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ jobs:
if: steps.iam-role.outcome == 'success'
shell: bash
env:
EXPECTED: 103
EXPECTED_MIN: 80
run: |
[[ $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l) -eq $EXPECTED ]] || { echo "Expected $EXPECTED recommendations but got $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l)"; exit 1; }
[[ $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l) -gt $EXPECTED_MIN ]] || { echo "Expected at least $EXPECTED_MIN recommendations but got $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l)"; exit 1; }
- name: Create file status_${{ matrix.os }}.txt and write the job status into it
if: always()
Expand Down Expand Up @@ -141,9 +141,9 @@ jobs:
if: steps.iam-role.outcome == 'success'
shell: bash
env:
EXPECTED: 16
EXPECTED_MIN: 10
run: |
[[ $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l) -eq $EXPECTED ]] || { echo "Expected $EXPECTED recommendations but got $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l)"; exit 1; }
[[ $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l) -gt $EXPECTED_MIN ]] || { echo "Expected at least $EXPECTED_MIN recommendations but got $(jq -r '.runs[0].results[].ruleId' code-guru/recommendations.sarif.json | wc -l)"; exit 1; }
- name: Create file status_${{ matrix.os }}.txt and write the job status into it
if: always()
Expand Down

0 comments on commit 2bc43bc

Please sign in to comment.