Skip to content

Commit

Permalink
Modify integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@vijaiaeroastro
vijaiaeroastro committed Jul 1, 2024
1 parent 481935d commit c6789a3
Showing 1 changed file with 65 additions and 6 deletions.
71 changes: 65 additions & 6 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -692,20 +692,39 @@ jobs:
cd test_suites
/usr/bin/time -v python integration_test.py 2>&1 | tee second_latest_sdk_test.log
- name: Compare results (Checks the total python script execution time)
- name: Compare results (Checks the total python script execution time and must pass/fail counts)
run: |
LATEST_TIME=$(grep "Elapsed (wall clock) time" test_suites/latest_sdk_test.log | awk '{print $8}')
SECOND_LATEST_TIME=$(grep "Elapsed (wall clock) time" test_suites/second_latest_sdk_test.log | awk '{print $8}')
LATEST_TOTAL_SECONDS=$(echo $LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
SECOND_LATEST_TOTAL_SECONDS=$(echo $SECOND_LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
echo "Latest SDK execution time in seconds: ${LATEST_TOTAL_SECONDS}"
echo "Second Latest SDK execution time in seconds: ${SECOND_LATEST_TOTAL_SECONDS}"
LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $1}')
SECOND_LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/second_latest_sdk_test.log | awk '{print $1}')
LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $1}')
SECOND_LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/second_latest_sdk_test.log | awk '{print $1}')
echo "Latest MUSTPASS: ${LATEST_MUSTPASS}"
echo "Second Latest MUSTPASS: ${SECOND_LATEST_MUSTPASS}"
echo "Latest MUSTFAIL: ${LATEST_MUSTFAIL}"
echo "Second Latest MUSTFAIL: ${SECOND_LATEST_MUSTFAIL}"
# Compare the total seconds
if (( $(echo "$LATEST_TOTAL_SECONDS < $SECOND_LATEST_TOTAL_SECONDS" | bc -l) )); then
echo "New release is better"
else
echo "New release is worse"
fi
if (( $(echo "$LATEST_TOTAL_SECONDS < $SECOND_LATEST_TOTAL_SECONDS" | bc -l) )); then
echo "New release is better in execution time"
else
echo "New release is worse in execution time"
fi
# Compare MUSTPASS and MUSTFAIL counts
if [ "$LATEST_MUSTPASS" != "$SECOND_LATEST_MUSTPASS" ] || [ "$LATEST_MUSTFAIL" != "$SECOND_LATEST_MUSTFAIL" ]; then
echo "MUSTPASS or MUSTFAIL counts have changed"
exit 1
else
echo "MUSTPASS and MUSTFAIL counts are consistent"
fi
integration-tests-latest-commit:
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -762,6 +781,23 @@ jobs:
LATEST_TOTAL_SECONDS=$(echo $LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
echo "Latest SDK execution time in seconds: ${LATEST_TOTAL_SECONDS}"
LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $1}')
LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $1}')
echo "MUSTPASS: ${LATEST_MUSTPASS}"
echo "MUSTFAIL: ${LATEST_MUSTFAIL}"
# Save counts for comparison in other jobs
echo "LATEST_MUSTPASS=${LATEST_MUSTPASS}" >> $GITHUB_ENV
echo "LATEST_MUSTFAIL=${LATEST_MUSTFAIL}" >> $GITHUB_ENV
# Fail the job if counts are inconsistent
if [ "$LATEST_MUSTPASS" != "$LATEST_MUSTFAIL" ]; then
echo "MUSTPASS or MUSTFAIL counts are inconsistent"
exit 1
else
echo "MUSTPASS and MUSTFAIL counts are consistent"
fi
integration-test-last-commit-and-last-release:
runs-on: ubuntu-20.04
needs: [set-integration-tests-status]
Expand Down Expand Up @@ -859,14 +895,37 @@ jobs:
LATEST_RELEASE_TOTAL_SECONDS=$(echo $LATEST_RELEASE_TIME | awk -F: '{ print ($1 * 60) + $2 }')
echo "Latest commit SDK execution time in seconds: ${LATEST_COMMIT_TOTAL_SECONDS}"
echo "Latest release SDK execution time in seconds: ${LATEST_RELEASE_TOTAL_SECONDS}"
LATEST_COMMIT_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_commit_sdk_test.log | awk '{print $1}')
LATEST_RELEASE_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_release_sdk_test.log | awk '{print $1}')
LATEST_COMMIT_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_commit_sdk_test.log | awk '{print $1}')
LATEST_RELEASE_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_release_sdk_test.log | awk '{print $1}')
echo "Latest commit MUSTPASS: ${LATEST_COMMIT_MUSTPASS}"

This comment has been minimized.

Sign in to view

Copy link
@gangatp

gangatp Jul 3, 2024

Collaborator

Can you also add must pass/ total must pass test cases and must fail/ total must fail test cases. Something like this
Uploading image.png…
echo "Latest release MUSTPASS: ${LATEST_RELEASE_MUSTPASS}"
echo "Latest commit MUSTFAIL: ${LATEST_COMMIT_MUSTFAIL}"
echo "Latest release MUSTFAIL: ${LATEST_RELEASE_MUSTFAIL}"
# Compare MUSTPASS and MUSTFAIL counts
if [ "$LATEST_COMMIT_MUSTPASS" != "$LATEST_RELEASE_MUSTPASS" ] || [ "$LATEST_COMMIT_MUSTFAIL" != "$LATEST_RELEASE_MUSTFAIL" ]; then
echo "MUSTPASS or MUSTFAIL counts have changed"
exit 1
else
echo "MUSTPASS and MUSTFAIL counts are consistent"
fi
# Compare the total seconds
if (( $(echo "$LATEST_COMMIT_TOTAL_SECONDS < $LATEST_RELEASE_TOTAL_SECONDS" | bc -l) )); then
echo "Latest commit is better"
else
echo "Latest release is better"
fi
echo "Latest commit with SHA ${{ github.sha }} ran in ${LATEST_COMMIT_TOTAL_SECONDS} seconds" > results.txt
echo "Latest release with tag ${{ env.LATEST_RELEASE_NAME }} ran in ${LATEST_RELEASE_TOTAL_SECONDS} seconds" >> results.txt
echo "Latest commit MUSTPASS: ${LATEST_COMMIT_MUSTPASS}" >> results.txt
echo "Latest release MUSTPASS: ${LATEST_RELEASE_MUSTPASS}" >> results.txt
echo "Latest commit MUSTFAIL: ${LATEST_COMMIT_MUSTFAIL}" >> results.txt
echo "Latest release MUSTFAIL: ${LATEST_RELEASE_MUSTFAIL}" >> results.txt
- name: Upload results artifact
uses: actions/upload-artifact@v4
Expand Down

0 comments on commit c6789a3

Please sign in to comment.