diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e1df4b1..681c8fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,29 +1,77 @@ -name: github-CI +name: main -on: [push] +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: jobs: - build: - + test: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, macOS-latest] - python-version: ["3.7", "3.8", "3.9", "3.10"] - name: Python ${{ matrix.python-version }} example + os: + - ubuntu-latest + - macos-latest + mpi: + - mpich + - openmpi + py: + # - "3.7" + # - "3.8" + # - "3.9" + - "3.10" + - "3.11" + - "3.12" steps: - - uses: actions/checkout@v3 - - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: setup-conda - uses: s-weigand/setup-conda@v1 + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 with: - update-conda: true - python-version: ${{ matrix.python-version }} - conda-channels: anaconda, conda-forge - - run: conda --version - - run: which python + python-version: ${{ matrix.py }} + - run: | - conda config --set always_yes yes - conda install -n root conda-build numpy fftw - conda build ./conf + # Install fftw + case $(uname) in + Linux) + sudo apt update + sudo apt install -y -q libfftw3-dev + ;; + Darwin) + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + brew install fftw + ;; + esac + + - run: python -m pip install -U pip build + + - run: python -m build + + - uses: mpi4py/setup-mpi@v1 + with: + mpi: ${{ matrix.mpi }} + + - run: pip install -vvv dist/mpi4py_fft-*.whl + env: + CFLAGS: "-O0" + + - run: pip install -r conf/requirements-test.txt + + - if: matrix.mpi == 'mpich' && startsWith(matrix.os, 'ubuntu') + run: ./runtests.sh + working-directory: tests + + - if: matrix.mpi == 'mpich' && startsWith(matrix.os, 'ubuntu') + uses: codecov/codecov-action@v4 + with: + files: test/coverage.xml + name: ${{ matrix.os }}-${{ matrix.mpi }}-${{ matrix.py }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 1d4d981..5c9f760 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ _templates/ *.dat *.html *.xml -*.txt *.h5 *.nc *.xdmf diff --git a/Makefile b/Makefile index ba41f89..ac24366 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION=$(shell python3 -c "import mpi4py_fft; print(mpi4py_fft.__version__)") default: - python setup.py build_ext -i + python setup.py build build_ext -i pip: rm -f dist/* @@ -15,6 +15,6 @@ tag: publish: tag pip clean: - git clean mpi4py_fft -fx + git clean -dxf mpi4py_fft cd docs && make clean && cd .. - @rm -rf *.egg-info/ build/ dist/ .eggs/ \ No newline at end of file + @rm -rf *.egg-info/ build/ dist/ .eggs/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 4d8c6d2..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,83 +0,0 @@ -jobs: -- job: macOS - displayName: macos-latest - pool: - vmImage: 'macos-latest' - strategy: - matrix: - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - - steps: - - bash: echo "##vso[task.prependpath]$CONDA/bin" - displayName: Add conda to PATH - - - bash: sudo chown -R $USER $CONDA - displayName: Take ownership of conda installation - - - bash: | - conda config --add channels conda-forge - conda config --set always_yes yes - conda install -n root conda-build numpy fftw - displayName: Set up Anaconda - - - bash: | - conda build --python $PYTHON_VERSION ./conf - displayName: Build and test - -- job: Ubuntu - displayName: Ubuntu - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - - steps: - - bash: echo "##vso[task.prependpath]$CONDA/bin" - displayName: Add conda to PATH - - - bash: | - conda config --add channels conda-forge - conda config --set always_yes yes - conda install -n root conda-build numpy fftw - displayName: Set up Anaconda - - - bash: | - conda build --no-test --python $PYTHON_VERSION ./conf - conda create --name mpi4py_fft_env mpi4py_fft_test coverage scipy pyfftw=0.12 python=$PYTHON_VERSION --use-local - source activate mpi4py_fft_env - pip install codecov - cd tests && ./runtests.sh - displayName: Build and test - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - - - bash: | - if [ $PYTHON_VERSION == 3.8 ]; then - source activate mpi4py_fft_env - cd tests - coverage xml -o ./coverage.xml - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -f coverage.xml - fi - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: Upload to Codecov diff --git a/conf/meta.yaml b/conf/meta.yaml index 8555074..84d4a10 100644 --- a/conf/meta.yaml +++ b/conf/meta.yaml @@ -16,7 +16,7 @@ requirements: - {{ compiler('cxx') }} host: - python - - cython <3.0 + - cython - numpy - pip - fftw @@ -25,7 +25,7 @@ requirements: run: - python - mpi4py - - mpich 4.0.2 + - mpich - {{ pin_compatible('numpy') }} - fftw - hdf5 * mpi_* diff --git a/conf/requirements-test.txt b/conf/requirements-test.txt new file mode 100644 index 0000000..8eadf40 --- /dev/null +++ b/conf/requirements-test.txt @@ -0,0 +1,3 @@ +coverage +pyfftw; sys_platform == 'linux' and python_version < '3.12' +scipy diff --git a/mpi4py_fft/fftw/fftw_xfftn.pxd b/mpi4py_fft/fftw/fftw_xfftn.pxd index aa095a0..2af0875 100644 --- a/mpi4py_fft/fftw/fftw_xfftn.pxd +++ b/mpi4py_fft/fftw/fftw_xfftn.pxd @@ -1,4 +1,5 @@ -cdef extern from "fftw3.h": +# cython: language_level=3str +cdef extern from "fftw3.h" nogil: ctypedef struct fftw_complex_struct: pass @@ -12,15 +13,15 @@ cdef extern from "fftw3.h": void fftw_destroy_plan(fftw_plan) - void fftw_execute_dft(fftw_plan, void *_in, void *_out) nogil + void fftw_execute_dft(fftw_plan, void *_in, void *_out) - void fftw_execute_dft_c2r(fftw_plan, void *_in, void *_out) nogil + void fftw_execute_dft_c2r(fftw_plan, void *_in, void *_out) - void fftw_execute_dft_r2c(fftw_plan, void *_in, void *_out) nogil + void fftw_execute_dft_r2c(fftw_plan, void *_in, void *_out) - void fftw_execute_r2r(fftw_plan, void *_in, void *_out) nogil + void fftw_execute_r2r(fftw_plan, void *_in, void *_out) - void fftw_execute(fftw_plan) nogil + void fftw_execute(fftw_plan) void fftw_init_threads() @@ -43,7 +44,7 @@ cdef extern from "fftw3.h": void fftw_print_plan(fftw_plan) -cdef extern from "fftw_planxfftn.h": +cdef extern from "fftw_planxfftn.h" nogil: ctypedef double fftw_real @@ -57,4 +58,4 @@ cdef extern from "fftw_planxfftn.h": int kind[], unsigned flags) -ctypedef void (*generic_function)(void *plan, void *_in, void *_out) nogil +ctypedef void (*generic_function)(void *plan, void *_in, void *_out) noexcept nogil diff --git a/mpi4py_fft/fftw/fftw_xfftn.pyx b/mpi4py_fft/fftw/fftw_xfftn.pyx index a84109e..859e4b5 100644 --- a/mpi4py_fft/fftw/fftw_xfftn.pyx +++ b/mpi4py_fft/fftw/fftw_xfftn.pyx @@ -1,5 +1,5 @@ -cimport fftw_xfftn -#cython: language_level=3 +# cython: language_level=3str +from . cimport fftw_xfftn cimport numpy as np from .utilities import * import numpy as np @@ -26,16 +26,16 @@ cpdef void cleanup(): fftw_cleanup() fftw_cleanup_threads() -cdef void _fftw_execute_dft(void *plan, void *_in, void *_out) nogil: +cdef void _fftw_execute_dft(void *plan, void *_in, void *_out) noexcept nogil: fftw_execute_dft(plan, _in, _out) -cdef void _fftw_execute_dft_r2c(void *plan, void *_in, void *_out) nogil: +cdef void _fftw_execute_dft_r2c(void *plan, void *_in, void *_out) noexcept nogil: fftw_execute_dft_r2c(plan, _in, _out) -cdef void _fftw_execute_dft_c2r(void *plan, void *_in, void *_out) nogil: +cdef void _fftw_execute_dft_c2r(void *plan, void *_in, void *_out) noexcept nogil: fftw_execute_dft_c2r(plan, _in, _out) -cdef void _fftw_execute_r2r(void *plan, void *_in, void *_out) nogil: +cdef void _fftw_execute_r2r(void *plan, void *_in, void *_out) noexcept nogil: fftw_execute_r2r(plan, _in, _out) cdef generic_function _get_execute_function(kind): diff --git a/mpi4py_fft/fftw/utilities.pyx b/mpi4py_fft/fftw/utilities.pyx index 66c5d71..cf3fd9c 100644 --- a/mpi4py_fft/fftw/utilities.pyx +++ b/mpi4py_fft/fftw/utilities.pyx @@ -1,4 +1,4 @@ -#cython: language_level=3 +#cython: language_level=3str cimport numpy as np import numpy as np diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3c963f7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools >= 42", "numpy", "cython >= 0.29.32"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 74309e3..cf8a287 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +"""mpi4py-fft -- Parallel Fast Fourier Transforms (FFTs) using MPI for Python""" import os import sys @@ -33,6 +34,8 @@ def get_prefix_dirs(): append(dirs, sys.prefix) if 'CONDA_BUILD' not in os.environ: append(dirs, '/usr') + append(dirs, '/usr/local') + append(dirs, '/opt/homebrew') return dirs def get_include_dirs(): @@ -76,7 +79,13 @@ def get_fftw_libs(): def generate_extensions(fftwlibs, force=True): """Generate files with float and long double""" - from distutils.dep_util import newer_group + try: + from setuptools.modified import newer_group + except ImportError: + try: + from setuptools.dep_util import newer_group + except ImportError: + from distutils.dep_util import newer_group for d in fftwlibs: if d == 'double': @@ -101,10 +110,10 @@ def generate_extensions(fftwlibs, force=True): def remove_extensions(fftwlibs): """Remove generated files""" for fname in ( - 'utilities.c', - 'fftw_xfftn.c', - 'fftwf_xfftn.c', - 'fftwl_xfftn.c', + 'utilities.c', + 'fftw_xfftn.c', + 'fftwf_xfftn.c', + 'fftwl_xfftn.c', ): dst = os.path.join(fftwdir, fname) try: @@ -116,10 +125,10 @@ def remove_extensions(fftwlibs): continue p = 'fftw'+prec_map[d]+'_' for fname in ( - 'fftw_planxfftn.h', - 'fftw_planxfftn.c', - 'fftw_xfftn.pyx', - 'fftw_xfftn.pxd', + 'fftw_planxfftn.h', + 'fftw_planxfftn.c', + 'fftw_xfftn.pyx', + 'fftw_xfftn.pxd', ): dst = os.path.join(fftwdir, fname.replace('fftw_', p)) try: @@ -131,20 +140,31 @@ def get_extensions(): """Return list of extension modules""" include_dirs = get_include_dirs() library_dirs = get_library_dirs() - ext = [Extension("mpi4py_fft.fftw.utilities", - sources=[os.path.join(fftwdir, "utilities.pyx")], - include_dirs=include_dirs)] + ext = [ + Extension( + "mpi4py_fft.fftw.utilities", + sources=[os.path.join(fftwdir, "utilities.pyx")], + define_macros=[('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')], + include_dirs=include_dirs, + ), + ] fftwlibs = get_fftw_libs() for d, libs in fftwlibs.items(): - p = 'fftw'+prec_map[d]+'_' - ext.append(Extension("mpi4py_fft.fftw.{}xfftn".format(p), - sources=[os.path.join(fftwdir, "{}xfftn.pyx".format(p)), - os.path.join(fftwdir, "{}planxfftn.c".format(p))], - #define_macros=[('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')], - libraries=libs, - include_dirs=include_dirs, - library_dirs=library_dirs)) + p = 'fftw' + prec_map[d] + '_' + ext.append( + Extension( + "mpi4py_fft.fftw.{}xfftn".format(p), + sources=[ + os.path.join(fftwdir, "{}xfftn.pyx".format(p)), + os.path.join(fftwdir, "{}planxfftn.c".format(p)), + ], + define_macros=[('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')], + libraries=libs, + include_dirs=include_dirs, + library_dirs=library_dirs, + ) + ) return ext @@ -190,14 +210,19 @@ def version(): if __name__ == '__main__': setup(name="mpi4py-fft", version=version(), - description="mpi4py-fft -- Parallel Fast Fourier Transforms (FFTs) using MPI for Python", + description=__doc__.strip(), long_description=long_description, + long_description_content_type='text/x-rst', author="Lisandro Dalcin and Mikael Mortensen", - url='https://github.com/mpi4py/mpi4py-fft', - packages=["mpi4py_fft", - "mpi4py_fft.fftw", - "mpi4py_fft.io"], - package_dir={"mpi4py_fft": "mpi4py_fft"}, + url="https://github.com/mpi4py/mpi4py-fft", + packages=[ + "mpi4py_fft", + "mpi4py_fft.fftw", + "mpi4py_fft.io", + ], + package_dir={ + "mpi4py_fft": "mpi4py_fft", + }, classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Console', @@ -209,10 +234,9 @@ def version(): 'License :: OSI Approved :: BSD License', 'Topic :: Scientific/Engineering :: Mathematics', 'Topic :: Software Development :: Libraries :: Python Modules', - ], + ], + keywords=['Python', 'FFTW', 'FFT', 'DCT', 'DST', 'MPI'], distclass=Dist, ext_modules=get_extensions(), install_requires=["mpi4py", "numpy"], - setup_requires=["setuptools>=18.0", "cython>=0.25"], - keywords=['Python', 'FFTW', 'FFT', 'DCT', 'DST', 'MPI'] ) diff --git a/tests/runtests.sh b/tests/runtests.sh index 341d36e..a95bc07 100755 --- a/tests/runtests.sh +++ b/tests/runtests.sh @@ -1,47 +1,39 @@ #!/bin/sh set -e -PY=$(python -c 'import sys; print(sys.version_info.major)') -export OMPI_MCA_plm=isolated -export OMPI_MCA_btl_vader_single_copy_mechanism=none -export OMPI_MCA_rmaps_base_oversubscribe=yes - -if [ $PY -eq 3 ]; then - # coverage only for python version 3 - - python -m coverage erase - - python -m coverage run -m test_fftw - python -m coverage run -m test_libfft - python -m coverage run -m test_io - python -m coverage run -m test_darray - mpiexec -n 2 python -m coverage run -m test_pencil - - #mpiexec -n 4 python -m coverage test_pencil.py - #mpiexec -n 8 python -m coverage test_pencil.py - mpiexec -n 2 python -m coverage run -m test_mpifft - #mpiexec -n 4 python -m coverage test_mpifft.py - # mpiexec -n 8 python -m coverage test_mpifft.py - # mpiexec -n 12 python -m coverage test_mpifft.py - mpiexec -n 2 python -m coverage run spectral_dns_solver.py - mpiexec -n 2 python -m coverage run -m test_io - #mpiexec -n 4 python -m coverage run -m test_io - mpiexec -n 2 python -m coverage run -m test_darray - #mpiexec -n 4 python -m coverage run -m test_darray - - python -m coverage combine - -else - python test_fftw.py - python test_libfft.py - mpiexec -n 2 python test_pencil.py - #mpiexec -n 4 python test_pencil.py - #mpiexec -n 8 python test_pencil.py - mpiexec -n 2 python test_mpifft.py - #mpiexec -n 4 python test_mpifft.py - # mpiexec -n 8 python test_mpifft.py - # mpiexec -n 12 python test_mpifft.py - mpiexec -n 2 python test_io.py - mpiexec -n 2 python test_darray.py - mpiexec -n 2 python spectral_dns_solver.py -fi +export OMPI_MCA_plm_ssh_agent=false +export OMPI_MCA_pml=ob1 +export OMPI_MCA_btl=tcp,self +export OMPI_MCA_mpi_yield_when_idle=true +export OMPI_MCA_btl_base_warn_component_unused=false +export OMPI_MCA_rmaps_base_oversubscribe=true +export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe + +set -x + +python -m coverage erase + +python -m coverage run -m test_fftw +python -m coverage run -m test_libfft +python -m coverage run -m test_io +python -m coverage run -m test_darray + +mpiexec -n 2 python -m coverage run -m test_pencil +mpiexec -n 4 python -m coverage run -m test_pencil +#mpiexec -n 8 python -m coverage test_pencil.py + +mpiexec -n 2 python -m coverage run -m test_mpifft +mpiexec -n 4 python -m coverage run -m test_mpifft +#mpiexec -n 8 python -m coverage test_mpifft.py +#mpiexec -n 12 python -m coverage test_mpifft.py + +mpiexec -n 2 python -m coverage run -m test_io +mpiexec -n 4 python -m coverage run -m test_io + +mpiexec -n 2 python -m coverage run -m test_darray +mpiexec -n 4 python -m coverage run -m test_darray + +mpiexec -n 2 python -m coverage run spectral_dns_solver.py + +python -m coverage combine +python -m coverage xml diff --git a/tests/test_mpifft.py b/tests/test_mpifft.py index 2ea851a..5d4ca30 100644 --- a/tests/test_mpifft.py +++ b/tests/test_mpifft.py @@ -152,8 +152,8 @@ def test_mpifft(): fft.backward.input_array.shape) assert (fft.backward.output_pencil.subshape == fft.backward.output_array.shape) - assert np.alltrue(np.array(fft.global_shape(True)) == np.array(fft.forward.output_pencil.shape)) - assert np.alltrue(np.array(fft.global_shape(False)) == np.array(fft.forward.input_pencil.shape)) + assert np.all(np.array(fft.global_shape(True)) == np.array(fft.forward.output_pencil.shape)) + assert np.all(np.array(fft.global_shape(False)) == np.array(fft.forward.input_pencil.shape)) ax = -1 if axes is None else axes[-1] if isinstance(axes[-1], int) else axes[-1][-1] assert fft.forward.input_pencil.substart[ax] == 0 assert fft.backward.output_pencil.substart[ax] == 0