Code coverage (#501)

* Delete unused yml file

* Reorganize and tidy `ci.yml`

* Change compilation flags and add switch for code coverage flags

* Fix and consolidate .gitignore files 

* Remove logfile printout

* Separate testing setup from main `Makefile`

* Update `make clean`

* Tidy makefiles

.github/workflows/ci.yml

@@ -9,93 +9,97 @@
 #
 # -----------------------------------------------------------------------------
 
-name: AtChem2-CI
+name: AtChem2 CI
 
 # ------------------------------ EVENTS ------------------------------ #
-# Controls when the action will run
+# Controls when the workflow is activated
 on:
-  # Triggers the workflow on push or pull request events but only for the master branch
+  # Triggers when a pull request is created or updated (only on the
+  # master branch)
   pull_request:
     branches: [ master ]
 
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-  # Triggers when pushed to the master (either by merging a Pull Request, or direct commit to master)
+  # Triggers when a push is made to the master branch (either by
+  # merging a pull request, or by direct commit)
   push:
     branches:
       - master
 
+  # Run manually from the Actions tab
+  workflow_dispatch:
+
 # ------------------------------ JOBS ------------------------------ #
-# A workflow run consists of one or more jobs that can run sequentially or in parallel
+# This workflow contains a single job called `testing`, which compiles
+# the AtChem2 model, runs the testsuite, and checks the testsuite code
+# coverage using Codecov
 jobs:
-  # This workflow contains a single job called `build`
-  build:
-    # Type of runner that the job will run on - we use a matrix to run on two OSs
-    # Runner images are described here: https://github.com/actions/runner-images
+  testing:
+    # The job runs on both linux (ubuntu) and macos runner images,
+    # with three versions of gnu fortran -- the runner images are
+    # described here: https://github.com/actions/runner-images
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-11, macos-12]
+        os: [ubuntu-22.04, macos-11] #, macos-12]
         fortran: [9, 10, 11]
-        exclude: #TODO: fortran9 does not work on macos12
-        - os: macos-12
-          fortran: 9
+        # exclude: #TODO: fortran9 does not work on macos12
+        # - os: macos-12
+        #   fortran: 9
       fail-fast: false
 
     # -------------------------------------------------------------
-    # Steps represent a sequence of tasks that will be executed as part of the job
+    # Sequence of tasks to be executed as part of the `testing` job:
+    # 1. checkout the repository
+    # 2. install gfortran, AtChem2 dependencies
+    # 3. compile AtChem2
+    # 4. run all tests (indent, style, unit, model)
+    # 5. recompile AtChem2, run unit and model tests for Codecov
     steps:
-      # Checkout the repository under $GITHUB_WORKSPACE, so the job can access it
+
+      # -------------------------------------------------------------
+      # (1) Checkout the repository under $GITHUB_WORKSPACE
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
       # -------------------------------------------------------------
-      # Install gfortran - if not supplied on the OS image
-      # macOS-11 only
-      - name: Install gfortran-9 on macOS-11
+      # (2) Install gfortran, if not supplied with the runner image,
+      # and the AtChem2 dependencies (CVODE, openlibm, numdiff, FRUIT)
+
+      # gfortran on macos-11
+      - name: Install gfortran-9 (macos-11)
         if: matrix.os == 'macos-11' && matrix.fortran == 9
         run: brew install gcc@${{ matrix.fortran }}
 
-      # macOS-12 only
-      - name: Install gfortran-9 or gfortran-10 on macOS-12
+      # gfortran on macos-12
+      - name: Install gfortran-9 and gfortran-10 (macos-12)
         if: matrix.os == 'macos-12' && (matrix.fortran == 9 || matrix.fortran == 10)
         run: brew install gcc@${{ matrix.fortran }}
 
-      # -------------------------------------------------------------
-      # Install cvode
-      # ubuntu only
-      - name: Install cvode (ubuntu)
-        if: startsWith(matrix.os, 'ubuntu')
+      # AtChem2 dependencies
+      - name: Install CVODE
         run: ./tools/install/install_cvode.sh $PWD $(which gfortran-${{ matrix.fortran }})
 
-      # macOS only
-      - name: Install cvode (macOS)
-        if: startsWith(matrix.os, 'macos')
-        run: ./tools/install/install_cvode.sh $PWD $(which gfortran-${{ matrix.fortran }})
-
-      # -------------------------------------------------------------
-      # Install all other dependencies
-      # both OSs
       - name: Install openlibm
         run: ./tools/install/install_openlibm.sh $PWD
 
       - name: Install numdiff
         run: ./tools/install/install_numdiff.sh $PWD
 
-      - name: Install fruit
+      - name: Install FRUIT
         run: sudo ./tools/install/install_fruit.sh $PWD
 
       # -------------------------------------------------------------
-      # Build AtChem2 and run all tests
+      # (3) Compile AtChem2 using the standard compilation flags
       - name: Build AtChem2
-        # Set FORT_VERSION for use inside the Makefile (called from build_atchem2.sh)
+        # Set $FORT_VERSION for use with the `Makefile`, called from
+        # the `build_atchem2.sh` script
         env:
           FORT_VERSION: ${{ matrix.fortran }}
         run: |
           cp tools/install/Makefile.skel Makefile
           ./build/build_atchem2.sh ./model/mechanism.fac
+          # macos only
           if [ $RUNNER_OS != "Linux" ] ; then
             install_name_tool -change libopenlibm.4.dylib $PWD/openlibm-0.8.1/libopenlibm.4.dylib atchem2 ;
             install_name_tool -change @rpath/libsundials_cvode.2.dylib $PWD/cvode/lib/libsundials_cvode.dylib atchem2 ;
@@ -104,31 +108,42 @@ jobs:
           fi
 
       # -------------------------------------------------------------
-      - name: Indent and style tests
+      # (4) Run all tests: indent, style, unit, model
+
+      - name: Indent and Style tests
         run: |
           make indenttest
           make styletest
 
-      # -------------------------------------------------------------
-      - name: Run AtChem2
+      - name: Unit and Model tests
+        # Set $FORT_VERSION for use with the `Makefile`
+        env:
+          FORT_VERSION: ${{ matrix.fortran }}
         run: |
-          # Run Atchem2
-          ./atchem2
-          # Upload `build` coverage to codecov
-          if [[ $RUNNER_OS == "Linux" ]]; then sudo ln -f -s /usr/bin/gcov-${{ matrix.fortran }} /usr/bin/gcov ; bash <(curl -s https://codecov.io/bash) -F build ; fi
+          export PATH=$PATH:$PWD/numdiff/bin
+          make unittests
+          make oldtests    # NB: oldtests will eventually be merged into modeltests
+          #make modeltests  # NB: modeltests are temporarily deactivated (pass on linux, fail on macos)
 
       # -------------------------------------------------------------
-      - name: Run tests
-        # Set FORT_VERSION for use inside the Makefile
+      # (5) Recompile AtChem2 using the code coverage flags, then
+      # upload the gcov files to Codecov
+
+      - name: Unit and Model tests (x Codecov)
+        if: matrix.os == 'ubuntu-22.04' && matrix.fortran == 11
+        # Set $FORT_VERSION for use with the `Makefile`
         env:
           FORT_VERSION: ${{ matrix.fortran }}
         run: |
-          # Run unit tests
           export PATH=$PATH:$PWD/numdiff/bin
-          make unittests
-          # Upload `unittests` coverage to codecov
-          if [[ $RUNNER_OS == "Linux" ]]; then sudo ln -f -s /usr/bin/gcov-${{ matrix.fortran }} /usr/bin/gcov ; bash <(curl -s https://codecov.io/bash) -F unittests ; fi
           make clean
-          # Run full build tests - this will upload `tests` coverage to codecov for each test
-          make oldtests
-          #make modeltests #TODO: modeltests are temporarily deactivated (pass on linux, fail on macos)
+          make unittests CCOV=true
+          mv tests/unit_tests/*.gc* ./  # Needed to deal with gcc11 changing the way it handles gcda and gcna files
+          make modeltests CCOV=true     # Run only the new model testsuite
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          #token: ${{ secrets.CODECOV_TOKEN }}
+          #directory: ./tests/unit_tests/
+          gcov: true

.gitignore

@@ -1,6 +1,7 @@
-# Ignore the executable, the Makefile, and the compiler-generated files in this directory
+# Ignore the executable, the Makefile, the Python bytecode, and the compiler-generated files
 atchem2
 Makefile
+*.pyc
 *.gcda
 *.gcno
 *.xml

tests/.gitignore

@@ -1,15 +1,16 @@
 # Ignore output/log files and mechanism files of the test runs
 *.log
 *.out
+*.output
 mechanism.f90
 mechanism.o
 mechanism.prod
 mechanism.reac
 mechanism.ro2
 mechanism.species
 mechanism.so
-tests/*/output/*
-model_tests/*/output/*
+tests/*/output/reactionRates/*
+model_tests/*/output/reactionRates/*
 
 # Except configuration and reference files
 !*.cmp

tests/makefile.tests

@@ -0,0 +1,84 @@
+# -----------------------------------------------------------------------------
+#
+# Copyright (c) 2009 - 2012 Chris Martin, Kasia Boronska, Jenny Young,
+# Peter Jimack, Mike Pilling
+#
+# Copyright (c) 2017 Sam Cox, Roberto Sommariva
+#
+# This file is part of the AtChem2 software package.
+#
+# This file is covered by the MIT license which can be found in the file
+# LICENSE.md at the top level of the AtChem2 distribution.
+#
+# -----------------------------------------------------------------------------
+
+# This is a secondary makefile, included in the main Makefile. It contains
+# the information needed to set up and run the AtChem2 Testsuite.
+#
+# The following variables are defined in the main Makefile:
+# - $FRUITDIR is the path to FRUIT, the FORTRAN Unit Test Framework
+# - $UNITTEST_SRCS are the Fortran source files needed for the unit tests
+# - $FORT_COMP is the Fortran compiler
+
+
+# ==================== Unit tests ==================== #
+
+# setup FRUIT
+UNITTESTDIR = tests/unit_tests
+fruit_code = $(FRUITDIR)/src/fruit.f90
+unittest_code = $(UNITTEST_SRCS) $(shell ls tests/unit_tests/*_test.f90 )
+unittest_code_gen = $(UNITTESTDIR)/fruit_basket_gen.f90 $(UNITTESTDIR)/fruit_driver_gen.f90
+all_unittest_code = $(fruit_code) $(unittest_code) $(unittest_code_gen)
+fruit_driver = $(UNITTESTDIR)/fruit_driver.exe
+
+# copy `fruit_generator.rb` to the unit tests directory and replace the
+# path to FRUIT with $FRUITDIR
+$(UNITTESTDIR)/fruit_basket_gen.f90 : $(unittest_code)
+	@echo ""
+	@cp tests/fruit_generator.rb $(UNITTESTDIR)
+	@cd $(UNITTESTDIR); \
+         sed -i "18s,.*,load \"$(FRUITDIR)/rake_base.rb\"," fruit_generator.rb; \
+         ruby fruit_generator.rb
+
+# build `fruit_driver.exe` from the individual unit tests
+$(fruit_driver) : $(all_unittest_code)
+	$(FORT_COMP) -o $(fruit_driver) -J$(OBJ) -I$(OBJ) $(all_unittest_code) $(FFLAGS) $(LDFLAGS)
+
+# ==================== Model tests  ==================== #
+
+# search `tests/tests/` for all subdirectories, which should reflect the full list of tests
+OLDTESTS := $(shell ls -d tests/tests/*/ | sed 's,tests/tests/,,g' | sed 's,/,,g')
+
+# search `tests/model_tests/` for all subdirectories, which should reflect the full list of model tests
+MODELTESTDIR = tests/model_tests
+MODELTESTS := $(shell ls -d tests/model_tests/*/ | sed 's,tests/model_tests/,,g' | sed 's,/,,g')
+
+# ==================== Makefile rules  ==================== #
+
+.PHONY: indenttest styletest unittests oldtests modeltests alltests
+
+indenttest:
+	@echo ""
+	@echo "Make: Running the indent test."
+	@./tests/run_indent_test.sh
+
+styletest:
+	@echo ""
+	@echo "Make: Running the style test."
+	@./tests/run_style_test.sh
+
+unittests: $(fruit_driver)
+	@export DYLD_LIBRARY_PATH=$(FORT_LIB):$(CVODELIBDIR):$(OPENLIBMDIR); $(fruit_driver)
+	@echo ""
+
+oldtests:
+	@echo ""
+	@echo "Make: Running the old tests:" $(OLDTESTS)
+	@./tests/run_tests.sh "$(OLDTESTS)" "$(FORT_LIB):$(CVODELIBDIR):$(OPENLIBMDIR)"
+
+modeltests:
+	@echo ""
+	@echo "Make: Running the model tests:" $(MODELTESTS)
+	@./tests/run_model_tests.sh "$(MODELTESTS)" "$(FORT_LIB):$(CVODELIBDIR):$(OPENLIBMDIR)"
+
+alltests: indenttest styletest oldtests modeltests unittests

tests/run_model_tests.sh

@@ -55,7 +55,7 @@ function find_string {
 # A mismatch generates a test failure. Numdiff is used to cope with small numerical
 # differences due to differing hardware, OS, and package versions.
 #
-# $2 is used to pass CVODELIB in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
+# $2 is used to pass $CVODELIBDIR in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
 #
 # $this_test_failures is used to keep a track of whether each test is passing: empty indicates
 # a pass, while non-empty indicates a failure.
@@ -215,8 +215,6 @@ $this_file_failures"
   echo $this_test_failures >> $LOG_FILE
 done
 
-if [[ "$RUNNER_OS" == "Linux" ]]; then bash <(curl -s https://codecov.io/bash) -F tests ; fi
-
 # After all tests are run, exit with a FAIL if $fail_counter>0, otherwise PASS.
 if [[ "$fail_counter" -gt 0 ]]; then
   echo "==> Model tests FAILED [" $fail_counter/$test_counter "]"
@@ -229,5 +227,5 @@ echo "" >> $LOG_FILE
 echo "Execution of model tests script finished." >> $LOG_FILE
 
 echo "==> Model tests logfile:" $LOG_FILE
-cat $LOG_FILE
+
 exit $model_tests_passed

tests/run_tests.sh

@@ -55,7 +55,7 @@ function find_string {
 # A mismatch generates a test failure. Numdiff is used to cope with small numerical
 # differences due to differing hardware, OS, and package versions.
 #
-# $2 is used to pass CVODELIB in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
+# $2 is used to pass $CVODELIBDIR in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
 #
 # $this_test_failures is used to keep a track of whether each test is passing: empty indicates
 # a pass, while non-empty indicates a failure.
@@ -215,8 +215,6 @@ $this_file_failures"
   echo $this_test_failures >> $LOG_FILE
 done
 
-if [[ "$RUNNER_OS" == "Linux" ]]; then bash <(curl -s https://codecov.io/bash) -F tests ; fi
-
 # After all tests are run, exit with a FAIL if $fail_counter>0, otherwise PASS.
 if [[ "$fail_counter" -gt 0 ]]; then
   echo "==> Tests FAILED [" $fail_counter/$test_counter "]"
@@ -225,5 +223,5 @@ else
   echo "==> Tests PASSED [" $test_counter/$test_counter "]"
   model_tests_passed=0
 fi
-cat $LOG_FILE
+
 exit $model_tests_passed