From a2ebfb622c1db19509cd2acf72096511112cd67c Mon Sep 17 00:00:00 2001 From: Nick Koukoufilippas Date: Wed, 15 Mar 2023 14:39:13 +0200 Subject: [PATCH] GHA & Installation improvements (#1025) * Improved GHA --------- Co-authored-by: Hsin Fan <57552401+hsinfan1996@users.noreply.github.com> --- .flake8 | 7 ++ .github/environment.yml | 14 +++ .github/workflows/ci.yml | 179 +++++++++++++++--------------- .gitignore | 5 +- README.md | 11 +- ci_scripts/install_class_linux.sh | 9 -- ci_scripts/install_class_osx.sh | 24 ---- cmake/Modules/BuildFFTW.cmake | 2 +- setup.cfg | 2 - setup.py | 11 +- 10 files changed, 124 insertions(+), 140 deletions(-) create mode 100644 .flake8 create mode 100644 .github/environment.yml delete mode 100755 ci_scripts/install_class_linux.sh delete mode 100755 ci_scripts/install_class_osx.sh delete mode 100644 setup.cfg diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..87a91cca8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,7 @@ +[flake8] +ignore = E741,E226,W503,W504 +exclude = + build/, + .eggs/, + benchmarks/data, + pyccl/ccllib.py diff --git a/.github/environment.yml b/.github/environment.yml new file mode 100644 index 000000000..02aeac4db --- /dev/null +++ b/.github/environment.yml @@ -0,0 +1,14 @@ +name: test # default testing environment name from conda-incubator +dependencies: + - pip + - cmake + - gsl + - swig + - pyyaml + - numpy + - scipy + - camb=1.3.6 + - isitgr + - fast-pt + - pytest + - pytest-cov diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84253658b..2d12628c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ name: continuous-integration + on: push: branches: @@ -7,129 +8,123 @@ on: - releases/* pull_request: null +env: + CACHE_NUMBER: 0 # increase to reset cache manually + CONDA_ENV: .github/environment.yml + jobs: build: runs-on: ${{ matrix.os }} defaults: run: shell: bash -l {0} + strategy: fail-fast: false matrix: - os: - - macos-11 - - ubuntu-latest + os: [macos-latest, ubuntu-latest] + include: + - os: macos-latest + label: osx-64 + prefix: /Users/runner/miniconda3 + + - os: ubuntu-latest + label: linux-64 + prefix: /usr/share/miniconda3 py: - 3.8 steps: - - name: cancel previous runs - uses: styfle/cancel-workflow-action@0.6.0 + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} - - uses: actions/checkout@v2 + - name: Checkout CCL repository + uses: actions/checkout@v3 - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up conda + uses: conda-incubator/setup-miniconda@v2 with: python-version: ${{ matrix.py }} - channels: conda-forge,defaults + channels: conda-forge, defaults channel-priority: strict show-channel-urls: true miniforge-version: latest miniforge-variant: Mambaforge - - name: remove homebrew - if: matrix.os == 'macos-11' + - name: "macOS: set env variables" + if: matrix.label == 'osx-64' run: | - curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall.sh -o uninstall_homebrew.sh - chmod u+x ./uninstall_homebrew.sh - ./uninstall_homebrew.sh -f -q >& /dev/null - rm -f uninstall_homebrew.sh + mamba install llvm-openmp + echo "DYLD_FALLBACK_LIBRARY_PATH=${CONDA_PREFIX}/lib" >> $GITHUB_ENV + SDKROOT=$(xcrun --sdk macosx --show-sdk-path) + echo "SDKROOT: ${SDKROOT}" + echo "SDKROOT=${SDKROOT}" >> $GITHUB_ENV + echo "CONDA_BUILD_SYSROOT=${SDKROOT}" >> $GITHUB_ENV - - name: lint - run: | - export MAMBA_NO_BANNER=1 - mamba install flake8 - flake8 pyccl - flake8 --exclude=data benchmarks - if [[ `grep "$(printf '\t')" pyccl/*.py` != "" ]]; then - exit 1 - fi - - - name: install deps - run: | - mamba install \ - pip \ - numpy nose coveralls pyyaml gsl fftw cmake swig scipy \ - compilers pkg-config setuptools_scm pytest pandas pytest-cov \ - cython "camb=1.3.6" isitgr traitlets fast-pt - - if [[ ${MATRIX_OS} == "macos-11" ]]; then - mamba install llvm-openmp - echo "DYLD_FALLBACK_LIBRARY_PATH=${CONDA_PREFIX}/lib" >> $GITHUB_ENV - SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - echo "SDKROOT: ${SDKROOT}" - echo "SDKROOT=${SDKROOT}" >> $GITHUB_ENV - echo "CONDA_BUILD_SYSROOT=${SDKROOT}" >> $GITHUB_ENV - fi - env: - MATRIX_OS: ${{ matrix.os }} - - - name: install class - run: | - if [[ ${MATRIX_OS} == "macos-11" ]]; then - export LDFLAGS="-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" - export LDFLAGS="$LDFLAGS -L/Users/runner/miniconda3/envs/test/lib" - . ci_scripts/install_class_osx.sh - else - . ci_scripts/install_class_linux.sh - fi - env: - MATRIX_OS: ${{ matrix.os }} - - - name: build CCL - run: | - python setup.py build - python setup.py develop + export CFLAGS="-I/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include" + export LDFLAGS="-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" - - name: c unit tests + - name: Set current date as environment variable + run: echo "TODAY=$(date +'%Y.%m.%d')" >> $GITHUB_ENV + + - name: Get cached environment + uses: actions/cache@v3 + id: cache + with: + path: ${{ matrix.prefix }} + key: + conda-py${{ matrix.py }}--${{ + matrix.os }}--${{ + hashFiles(env.CONDA_ENV) }}-${{ + hashFiles('setup.py') }}--${{ + env.TODAY }}-${{ + env.CACHE_NUMBER }} + + - name: Install environment packages + if: steps.cache.outputs.cache-hit != 'true' run: | - cd build - make -j4 - CLASS_PARAM_DIR=./extern/share/class/ OMP_NUM_THREADS=2 ./check_ccl + export MAMBA_NO_BANNER=1 # hide mamba banner from output + mamba env update --file ${{ env.CONDA_ENV }} --prune + pip install "classy<3" - - name: python unit tests + - name: Install CCL run: | - OMP_NUM_THREADS=2 pytest -vv pyccl --cov=pyccl + pip install -e . - - name: benchmarks + - name: Unit tests run: | - OMP_NUM_THREADS=2 pytest -vv benchmarks --cov=pyccl --cov-append + OMP_NUM_THREADS=2 pytest -vv pyccl --cov=pyccl --cov-report=lcov:lcov.info - - name: coveralls - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Benchmarks run: | - coveralls --service=github - -# build-doc-only-ubuntu: -# runs-on: ubuntu-latest -# -# steps: -# - uses: actions/checkout@v2 -# - name: install latex -# run: | -# -# sudo apt install \ -# texlive texlive-bibtex-extra texlive-science texlive-publishers \ -# latexmk python3-sphinx python3-sphinx-rtd-theme python3-nbconvert \ -# python3-jupyter-client jupyter-client jupyter-nbconvert sphinx-common \ -# pandoc python3-setuptools \ -# -y -# sudo pip3 install mkauthlist -# -# - name: build docs -# run: | -# cd doc/0000-ccl_note -# make + OMP_NUM_THREADS=2 pytest -vv benchmarks --cov=pyccl --cov-report=lcov:lcov.info --cov-append + + - name: Code coverage + if: matrix.label == 'linux-64' + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: lcov.info + + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout CCL repository + uses: actions/checkout@v3 + + - name: Lint + uses: py-actions/flake8@v2 + with: + args: "--config .flake8" + + +# Legacy step: uninstall homebrew (CI works without it). +# - name: "macOS: remove homebrew" +# if: matrix.label == 'osx-64' +# run: | +# curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall.sh -o uninstall_homebrew.sh +# chmod u+x ./uninstall_homebrew.sh +# ./uninstall_homebrew.sh -f -q >& /dev/null +# rm -f uninstall_homebrew.sh diff --git a/.gitignore b/.gitignore index e371957ab..ae7d2edf5 100644 --- a/.gitignore +++ b/.gitignore @@ -59,8 +59,9 @@ tests/benchmark/codes/.ipynb_checkpoints/* # Ignore rope project .ropeproject -# Ignore the automatically generated coverage file -.coverage +# Ignore the automatically generated coverage files +.coverage* +*cov.info # Ignore DS_Store file .DS_Store diff --git a/README.md b/README.md index 38c0d515f..388e8287b 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,10 @@ or $ pip install pyccl ``` -For the PyPi installation, you will need ``CMake`` installed locally. See -[Getting CMake](https://ccl.readthedocs.io/en/latest/source/installation.html#getting-cmake) -for instructions. Note that the code only supports Linux or Mac OS, but no Windows. +For the PyPi installation, you will need ``CMake`` and ``SWIG`` installed locally. +See [Getting CMake](https://ccl.readthedocs.io/en/latest/source/installation.html#getting-cmake) +and [Installing SWIG](https://pypi.org/project/swig/) for instructions. +Note that the code only supports Linux or Mac OS, but no Windows. Once you have the code installed, you can take it for a spin! @@ -81,7 +82,7 @@ lens2 = ccl.WeakLensingTracer(cosmo, dndz=(z_n, n)) # Calculate the angular cross-spectrum of the two tracers as a function of ell ell = np.arange(2, 10) -cls = ccl.angular_cl(cosmo, lens1, lens2, ell) +cls = cosmo.angular_cl(lens1, lens2, ell) print(cls) ``` @@ -113,7 +114,7 @@ for more information. If you have comments, questions, or feedback, please [write us an issue](https://github.com/LSSTDESC/CCL/issues). -The current lead of the LSST DESC CCL Topical Team is David Alonso (damonge, david.alonso at physics.ox.ac.uk) +The current lead of the LSST DESC CCL Topical Team is David Alonso (damonge, david.alonso at physics.ox.ac.uk). # Acknowledgements diff --git a/ci_scripts/install_class_linux.sh b/ci_scripts/install_class_linux.sh deleted file mode 100755 index 1b1a0c0bd..000000000 --- a/ci_scripts/install_class_linux.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -git clone https://github.com/lesgourg/class_public.git -cd class_public -git checkout v2.7.2 -make -j4 - -# at this point the make file leaves you in the python dir -cd .. diff --git a/ci_scripts/install_class_osx.sh b/ci_scripts/install_class_osx.sh deleted file mode 100755 index eb7cb351d..000000000 --- a/ci_scripts/install_class_osx.sh +++ /dev/null @@ -1,24 +0,0 @@ -export CONDA_BUILD_SYSROOT=/ - -rm -rf class_public -git clone --depth=1000 https://github.com/lesgourg/class_public.git -cd class_public -git checkout v2.7.2 - -echo "----------------------------------" -env -echo "----------------------------------" - -sed -i.bak -e 's/^CC/#CC/g' Makefile -sed -i.bak -e 's/^OPTFLAG =/OPTFLAG = ${CFLAGS} ${LDFLAGS}/g' Makefile -sed -i.bak -e 's/^#CCFLAG +=/CCFLAG +=/g' Makefile -sed -i.bak -e 's/^#CCFLAG =/CCFLAG =/g' Makefile - -echo "----------------------------------" -cat Makefile -echo "----------------------------------" - -make -j4 - -# at this point the make file leaves you in the python dir -cd .. diff --git a/cmake/Modules/BuildFFTW.cmake b/cmake/Modules/BuildFFTW.cmake index 377a1fd14..fafcb2eb7 100644 --- a/cmake/Modules/BuildFFTW.cmake +++ b/cmake/Modules/BuildFFTW.cmake @@ -5,7 +5,7 @@ set(FFTWMD5 0d5915d7d39b3253c1cc05030d79ac47) find_package(FFTW) -# If GSL is not installed, lets go ahead and compile it +# If FFTW is not installed, lets go ahead and compile it if(NOT FFTW_FOUND ) message(STATUS "FFTW not found, downloading and compiling from source") ExternalProject_Add(FFTW diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b59e7f5a0..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[flake8] -ignore = E741,E226,W503,W504 diff --git a/setup.py b/setup.py index 0b937ca41..d9055b6bf 100644 --- a/setup.py +++ b/setup.py @@ -92,18 +92,19 @@ def run(self): include_package_data=True, use_scm_version=True, distclass=Distribution, - python_requires='>=3.7.0', + python_requires='>=3.7', setup_requires=['setuptools_scm'], - install_requires=['numpy', 'pyyaml'], + install_requires=['numpy'], cmdclass={'build_py': Build, 'develop': Develop, 'clean': Clean}, classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Science/Research', - 'License :: OSI Approved', + 'License :: OSI Approved :: BSD License', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX :: Linux', 'Programming Language :: C', 'Programming Language :: Python', - 'Topic :: Scientific/Engineering :: Physics' + 'Topic :: Scientific/Engineering :: Physics', + 'Topic :: Scientific/Engineering :: Astronomy', ] )