[ci] Add test coverage check

Signed-off-by: Andrew Shkrob <[email protected]>

.github/workflows/coverage-check.yaml

@@ -0,0 +1,144 @@
+name: Coverage Report
+on:
+  workflow_dispatch: # Manual trigger
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - labeled
+      - unlabeled
+
+# Cancels previous jobs if the same branch or PR was updated again.
+concurrency:
+  group: ${{ github.workflow }}-coverage-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  should-run-check:
+    name: Should run coverage
+    runs-on: ubuntu-22.04
+    outputs:
+      run-from-pr: ${{ steps.run-from-pr.outputs.run-from-pr }}
+      manually-triggered: ${{ steps.manually-triggered.outputs.manually-triggered }}
+    steps:
+      - name: Check if PR has 'Coverage' label
+        id: run-from-pr
+        if: github.event_name == 'pull_request'
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          LABEL_NAME="Coverage"
+          LABELS=$(gh pr view https://github.com/$GITHUB_REPOSITORY/pull/$PR_NUMBER --json labels)
+          if echo "$LABELS" | jq -e '.labels[].name' | grep -q "$LABEL_NAME"; then
+            echo "run-from-pr=true" >> $GITHUB_OUTPUT
+            echo "'Coverage' label found in PR."
+          fi
+      - name: Check if manually triggered
+        id: manually-triggered
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "manually-triggered=true" >> $GITHUB_OUTPUT
+
+  coverage:
+    needs: should-run-check
+    name: Generate coverage report
+    runs-on: ubuntu-22.04
+    if: ${{ needs.should-run-check.outputs.run-from-pr == 'true' || needs.should-run-check.outputs.manually-triggered == 'true'}}
+    steps:
+      - name: Free disk space by removing .NET, Android and Haskell
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100 # enough to get all commits for the current day
+
+      - name: Parallel submodules checkout
+        shell: bash
+        run: git submodule update --depth 1 --init --recursive --jobs=$(($(nproc) * 20))
+
+      - name: Install build tools and dependencies
+        shell: bash
+        run: |
+          sudo apt update -y
+          sudo apt install -y \
+              ninja-build \
+              libgl1-mesa-dev \
+              libglvnd-dev \
+              qt6-base-dev \
+              libqt6svg6-dev \
+              qt6-positioning-dev \
+              libqt6positioning6-plugins \
+              libqt6positioning6 \
+              llvm
+          pip install gcovr
+
+      - name: Configure
+        shell: bash
+        run: ./configure.sh
+
+      - name: Configure ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ github.workflow }}-coverage
+
+      - name: CMake
+        shell: bash
+        env:
+          CC: clang-14
+          CXX: clang++-14
+          CMAKE_C_COMPILER_LAUNCHER: ccache
+          CMAKE_CXX_COMPILER_LAUNCHER: ccache
+        # -g1 should slightly reduce build time.
+        run: |
+          cmake . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_CXX_FLAGS=-g1 -DCOVERAGE_REPORT=ON
+
+      - name: Compile
+        shell: bash
+        working-directory: build
+        run: ninja
+
+      - name: Tests
+        shell: bash
+        working-directory: build
+        env:
+          # drape_tests - requires X Window
+          # generator_integration_tests - https://github.com/organicmaps/organicmaps/issues/225
+          # opening_hours_integration_tests - https://github.com/organicmaps/organicmaps/issues/219
+          # opening_hours_supported_features_tests - https://github.com/organicmaps/organicmaps/issues/219
+          # routing_integration_tests - https://github.com/organicmaps/organicmaps/issues/221
+          # shaders_tests - https://github.com/organicmaps/organicmaps/issues/223
+          # world_feed_integration_tests - https://github.com/organicmaps/organicmaps/issues/215
+          CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests"
+        run: |
+          sudo locale-gen en_US
+          sudo locale-gen en_US.UTF-8
+          sudo locale-gen es_ES
+          sudo locale-gen es_ES.UTF-8
+          sudo locale-gen fr_FR
+          sudo locale-gen fr_FR.UTF-8
+          sudo locale-gen ru_RU
+          sudo locale-gen ru_RU.UTF-8
+          sudo update-locale
+          ln -s ../data data
+          ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
+
+      - name: Run coverage report generation
+        shell: bash
+        working-directory: build
+        run: |
+          cmake --build . --target omim_coverage
+          cat coverage_report/summary.txt
+
+      - name: Archive the coverage report
+        working-directory: build/coverage_report
+        run: zip -r coverage_report.zip html/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage-report
+          path: build/coverage_report/coverage_report.zip

.github/workflows/linux-check.yaml

@@ -184,4 +184,4 @@ jobs:
           sudo locale-gen ru_RU.UTF-8
           sudo update-locale
           ln -s ../data data
-          ctest -LE "fixture" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
+          ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure

.github/workflows/macos-check.yaml

@@ -95,4 +95,4 @@ jobs:
           CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests"
         run: |
           ln -s ../data data
-          ctest -LE "fixture" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
+          ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure

CMakeLists.txt

@@ -22,6 +22,8 @@ endif()
 
 message(STATUS "Using compiler ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
 
+option(COVERAGE_REPORT "Configure for coverage report" OFF)
+
 option(UNITY_DISABLE "Disable unity build" OFF)
 if (NOT UNITY_DISABLE AND NOT DEFINED ENV{UNITY_DISABLE})
   set(CMAKE_UNITY_BUILD ON)
@@ -212,6 +214,9 @@ if (NOT SKIP_TESTS)
   enable_testing()
   # Enables ctest -T memcheck with valgrind
   include(CTest)
+  if (COVERAGE_REPORT)
+    include(OmimCoverage)
+  endif ()
 endif()
 
 if (NOT PYTHON_VERSION)

cmake/OmimCoverage.cmake

@@ -0,0 +1,41 @@
+add_compile_options(-O0 --coverage)
+add_link_options(--coverage)
+
+set(COVERAGE_REPORT_DIR ${CMAKE_BINARY_DIR}/coverage_report)
+
+find_program(GCOVR_EXECUTABLE_PATH gcovr)
+if (NOT GCOVR_EXECUTABLE_PATH)
+  message(FATAL_ERROR "'gcovr' is required to generate test coverage report. Details: gcovr.com.")
+endif ()
+
+if (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU|AppleClang")
+  set(GCOV_EXECUTABLE "gcov")
+elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+  set(GCOV_EXECUTABLE "llvm-cov")
+endif ()
+
+find_program(GCOV_EXECUTABLE_PATH ${GCOV_EXECUTABLE})
+if (NOT GCOV_EXECUTABLE_PATH)
+  message(FATAL_ERROR "'${GCOV_EXECUTABLE}' is required to generate test coverage report.")
+endif ()
+
+if (${GCOV_EXECUTABLE_PATH} MATCHES "llvm-cov")
+  set(GCOV_EXECUTABLE_PATH "${GCOV_EXECUTABLE_PATH} gcov")
+endif ()
+
+add_custom_target(omim_coverage
+  # Remove harfbuzz.cc.* files because they reference .rl files that do not exist and cannot be excluded by gcovr.
+  COMMAND rm -f ${CMAKE_BINARY_DIR}/3party/harfbuzz/CMakeFiles/harfbuzz.dir/harfbuzz/src/harfbuzz.cc.*
+  # Recreate coverage_report folder
+  COMMAND rm -rf ${COVERAGE_REPORT_DIR} && mkdir ${COVERAGE_REPORT_DIR}
+  # Run gcovr
+  COMMAND ${GCOVR_EXECUTABLE_PATH}
+  --config=${OMIM_ROOT}/gcovr.cfg
+  --root=${OMIM_ROOT}
+  --object-directory=${CMAKE_BINARY_DIR}
+  --exclude=${CMAKE_BINARY_DIR} # Exclude autogenerated files from Qt and some 3party libraries.
+  --gcov-executable=${GCOV_EXECUTABLE_PATH}
+  --html-nested=${COVERAGE_REPORT_DIR}/html/ >> ${COVERAGE_REPORT_DIR}/summary.txt
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  COMMENT "Generating coverage report..."
+)

cmake/OmimTesting.cmake

@@ -13,7 +13,7 @@ add_test(
 )
 set_tests_properties(OmimStartTestServer PROPERTIES FIXTURES_SETUP TestServer)
 set_tests_properties(OmimStopTestServer PROPERTIES FIXTURES_CLEANUP TestServer)
-set_tests_properties(OmimStartTestServer OmimStopTestServer PROPERTIES LABELS "fixture")
+set_tests_properties(OmimStartTestServer OmimStopTestServer PROPERTIES LABELS "omim-fixture")
 
 # Options:
 # * REQUIRE_QT - requires QT event loop
@@ -80,6 +80,7 @@ function(omim_add_ctest name require_server boost_test)
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
   )
   if (require_server)
-    set_tests_properties(${TEST_NAME} PROPERTIES FIXTURES_REQUIRED TestServer)
+    set_tests_properties(${name} PROPERTIES FIXTURES_REQUIRED TestServer)
   endif()
+  set_tests_properties(${name} PROPERTIES LABELS "omim-test")
 endfunction()

docs/INSTALL.md

@@ -254,15 +254,15 @@ Run all unit tests:
 
 ```bash
 cd build
-ctest -LE "fixture" --output-on-failure
+ctest -L "omim_test" --output-on-failure
 ```
 
 To run a limited set of tests, use `-R <regex>` flag. To exclude some tests, use `-E <regex>` flag:
 
 ```bash
 cd build
 ctest -R "base_tests|coding_tests" --output-on-failure
-ctest -LE "fixture" -E "base_tests|coding_tests" --output-on-failure
+ctest -L "omim_test" -E "base_tests|coding_tests" --output-on-failure
 ```
 
 When developing, it is more convenient to use a symlink:
@@ -275,6 +275,53 @@ ln -s ../data/ data
 
 Some tests [are known to be broken](https://github.com/organicmaps/organicmaps/issues?q=is%3Aissue+is%3Aopen+label%3ATests) and disabled on CI.
 
+### Test Coverage
+
+To generate a test coverage report you'll need [gcovr](https://gcovr.com) and gcov tools installed.
+
+Installing gcovr on Linux:
+```bash
+pip3 install gcovr
+```
+
+Installing gcovr on MacOS:
+```bash
+brew install gcovr
+```
+
+Installing gcov on Linux:
+```bash
+# If you're using GCC compiler
+sudo apt-get install cpp
+
+# If you're using Clang compiler
+sudo apt-get install llvm
+```
+
+Installing gcov on MacOS:
+```bash
+# If you're using AppleClang compiler it should already be installed
+
+# If you're using Clang compiler
+brew install llvm
+```
+
+Steps to generate coverage report:
+
+1. Configure cmake with `-DCOVERAGE_REPORT=ON` flag:
+   ```bash
+   cmake . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug \
+           -DCMAKE_CXX_FLAGS=-g1 -DCOVERAGE_REPORT=ON
+   ```
+2. Compile unit tests.
+3. Run unit tests.
+4. Run coverage report generation:
+   ```bash
+   cd build
+   cmake --build . --target omim_coverage
+   ```
+5. Report can be found in the `build/coverage_report` folder.
+
 ### Debug commands
 
 Organic Maps has some "hidden" debug commands that you can trigger by entering them into the search box.

gcovr.cfg

@@ -0,0 +1,22 @@
+# gcovr configuration file
+# Details: https://gcovr.com/en/master/manpage.html
+
+# Exclude all subfolders of "3party/" except for "opening_hours"
+exclude = ^3party\/(?!opening_hours($|\/)).*
+# Exclude testing folders
+exclude = testing/
+exclude = qt_tstfrm/
+# Exclude all "*tests*" subfolders
+exclude = .*tests.*
+
+exclude-unreachable-branches = yes
+exclude-throw-branches = yes
+
+print-summary = yes
+
+gcov-parallel = 20
+gcov-ignore-errors = all
+gcov-ignore-parse-errors = all
+
+html-title = Organic Maps Code Coverage Report
+html-self-contained = yes