🧪 Upload ansible-test coverage to Codecov

.github/workflows/ci.yml

@@ -192,8 +192,50 @@ jobs:
  run: python3 -m pip install --upgrade ansible-core
 
  - name: Run sanity tests
+ id: make-run
  run: make test_collection_sanity
 
+ - name: Upload test coverage to Codecov
+ if: >-
+ !cancelled()
+ && steps.make-run.outputs.cov-report-files != ''
+ uses: codecov/codecov-action@v4
+ with:
+ fail_ci_if_error: >-
+ ${{
+ toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+ }}
+ files: >-
+ ${{ steps.make-run.outputs.cov-report-files }}
+ flags: >-
+ CI-GHA,
+ ansible-test,
+ sanity,
+ OS-${{
+ runner.os
+ }}
+ token: ${{ secrets.CODECOV_TOKEN }}
+ - name: Upload test results to Codecov
+ if: >-
+ !cancelled()
+ && steps.make-run.outputs.test-result-files != ''
+ uses: codecov/test-results-action@v1
+ with:
+ fail_ci_if_error: >-
+ ${{
+ toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+ }}
+ files: >-
+ ${{ steps.make-run.outputs.test-result-files }}
+ flags: >-
+ CI-GHA,
+ ansible-test,
+ sanity,
+ OS-${{
+ runner.os
+ }}
+ token: ${{ secrets.CODECOV_TOKEN }}
+
  collection-integration:
  name: awx_collection integration
  runs-on: ubuntu-latest
@@ -225,17 +267,39 @@ jobs:
  python3 -m pip install -r awx_collection/requirements.txt
 
  - name: Run integration tests
+ id: make-run
  run: |
  echo "::remove-matcher owner=python::" # Disable annoying annotations from setup-python
  echo '[general]' > ~/.tower_cli.cfg
  echo 'host = https://${{ steps.awx.outputs.ip }}:8043' >> ~/.tower_cli.cfg
  echo 'oauth_token = ${{ steps.awx.outputs.admin-token }}' >> ~/.tower_cli.cfg
  echo 'verify_ssl = false' >> ~/.tower_cli.cfg
  TARGETS="$(ls awx_collection/tests/integration/targets | grep '${{ matrix.target-regex.regex }}' | tr '\n' ' ')"
- make COLLECTION_VERSION=100.100.100-git COLLECTION_TEST_TARGET="--coverage --requirements $TARGETS" test_collection_integration
+ make COLLECTION_VERSION=100.100.100-git COLLECTION_TEST_TARGET="--requirements $TARGETS" test_collection_integration
  env:
  ANSIBLE_TEST_PREFER_PODMAN: 1
 
+ - name: Upload test coverage to Codecov
+ if: >-
+ !cancelled()
+ && steps.make-run.outputs.cov-report-files != ''
+ uses: codecov/codecov-action@v4
+ with:
+ fail_ci_if_error: >-
+ ${{
+ toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+ }}
+ files: >-
+ ${{ steps.make-run.outputs.cov-report-files }}
+ flags: >-
+ CI-GHA,
+ ansible-test,
+ integration,
+ OS-${{
+ runner.os
+ }}
+ token: ${{ secrets.CODECOV_TOKEN }}
+
  # Upload coverage report as artifact
  - uses: actions/upload-artifact@v3
  if: always()

Makefile

@@ -23,7 +23,7 @@ COLLECTION_TEST_TARGET ?=
 # args for collection install
 COLLECTION_PACKAGE ?= awx
 COLLECTION_NAMESPACE ?= awx
-COLLECTION_INSTALL = ~/.ansible/collections/ansible_collections/$(COLLECTION_NAMESPACE)/$(COLLECTION_PACKAGE)
+COLLECTION_INSTALL = $(HOME)/.ansible/collections/ansible_collections/$(COLLECTION_NAMESPACE)/$(COLLECTION_PACKAGE)
 COLLECTION_TEMPLATE_VERSION ?= false
 
 # NOTE: This defaults the container image version to the branch that's active
@@ -412,10 +412,23 @@ test_collection_sanity:
  if ! [ -x "$(shell command -v ansible-test)" ]; then pip install ansible-core; fi
  ansible --version
  COLLECTION_VERSION=1.0.0 $(MAKE) install_collection
- cd $(COLLECTION_INSTALL) && ansible-test sanity $(COLLECTION_SANITY_ARGS)
+ cd $(COLLECTION_INSTALL) && \
+ ansible-test sanity --coverage --junit $(COLLECTION_SANITY_ARGS) && \
+ ansible-test coverage xml $(COLLECTION_SANITY_ARGS) --group-by command --group-by version
+ @if [ "${GITHUB_ACTIONS}" = "true" ]; \
+ then \
+ echo cov-report-files="$(shell ls -m $(COLLECTION_INSTALL)/tests/output/reports/coverage=sanity*.xml)" >> "${GITHUB_OUTPUT}"; \
+ echo test-result-files="$(shell ls -m $(COLLECTION_INSTALL)/tests/output/junit/*.xml)" >> "${GITHUB_OUTPUT}"; \
+ fi
 
 test_collection_integration: install_collection
- cd $(COLLECTION_INSTALL) && ansible-test integration -vvv $(COLLECTION_TEST_TARGET)
+ cd $(COLLECTION_INSTALL) && \
+ ansible-test integration --coverage -vvv $(COLLECTION_TEST_TARGET) && \
+ ansible-test coverage xml --requirements --group-by command --group-by version
+ @if [ "${GITHUB_ACTIONS}" = "true" ]; \
+ then \
+ echo cov-report-files="$(shell ls -m $(COLLECTION_INSTALL)/tests/output/reports/coverage=integration*.xml)" >> "${GITHUB_OUTPUT}"; \
+ fi
 
 test_unit:
  @if [ "$(VENV_BASE)" ]; then \