From 00127cc966d81515102ccf2404d737a0d0758012 Mon Sep 17 00:00:00 2001 From: Connor Ward Date: Fri, 20 Dec 2024 10:26:40 +0000 Subject: [PATCH] Use a pip-installed libsupermesh (#3920) --- .github/workflows/build-mac.yml | 3 ++ .github/workflows/build.yml | 1 + .github/workflows/pip-mac.yml | 22 ++-------- .github/workflows/pip.yml | 19 --------- .github/workflows/pyop2.yml | 34 --------------- docker/Dockerfile.complex | 1 + docker/Dockerfile.vanilla | 3 +- docs/source/download.rst | 1 - docs/source/parallelism.rst | 5 ++- firedrake/supermeshing.py | 6 ++- pyproject.toml | 2 + requirements-git.txt | 1 + scripts/firedrake-install | 74 +++++++++++++-------------------- setup.py | 20 +++++---- 14 files changed, 61 insertions(+), 131 deletions(-) diff --git a/.github/workflows/build-mac.yml b/.github/workflows/build-mac.yml index 51aa9f5104..f8530311df 100644 --- a/.github/workflows/build-mac.yml +++ b/.github/workflows/build-mac.yml @@ -50,6 +50,9 @@ jobs: run: | . ../firedrake_venv/bin/activate python -m pytest -v tests/firedrake/regression/ -k "poisson_strong or stokes_mini or dg_advection" + # also test for 'problem libraries' (spatialindex and libsupermesh) + python -m pytest -v tests/firedrake/regression/test_locate_cell.py + python -m pytest -v tests/firedrake/supermesh/test_assemble_mixed_mass_matrix.py timeout-minutes: 30 - name: Post-run cleanup if: ${{ always() }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 909db6990a..0eb616c24d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,6 +66,7 @@ jobs: --mpicxx="$MPICH_DIR"/mpicxx \ --mpif90="$MPICH_DIR"/mpif90 \ --mpiexec="$MPICH_DIR"/mpiexec \ + --mpihome="$MPICH_DIR"/.. \ --venv-name firedrake_venv \ --no-package-manager \ --disable-ssh \ diff --git a/.github/workflows/pip-mac.yml b/.github/workflows/pip-mac.yml index c8d1fbfe2e..54e2ada7cc 100644 --- a/.github/workflows/pip-mac.yml +++ b/.github/workflows/pip-mac.yml @@ -68,25 +68,6 @@ jobs: --download-superlu_dist make - - name: Install libsupermesh - run: | - source pip_venv/bin/activate - python -m pip install 'rtree>=1.2' - cd pip_venv/src - git clone https://github.com/firedrakeproject/libsupermesh.git - mkdir -p libsupermesh/build - cd libsupermesh/build - cmake .. \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX="$VIRTUAL_ENV" \ - -DMPI_C_COMPILER=/opt/homebrew/bin/mpicc \ - -DMPI_CXX_COMPILER=/opt/homebrew/bin/mpicxx \ - -DMPI_Fortran_COMPILER=/opt/homebrew/bin/mpif90 \ - -DCMAKE_Fortran_COMPILER=/opt/homebrew/bin/mpif90 \ - -DMPIEXEC_EXECUTABLE=/opt/homebrew/bin/mpiexec - make - make install - - uses: actions/checkout@v4 with: path: pip_venv/src/firedrake @@ -118,6 +99,9 @@ jobs: cd pip_venv/src/firedrake python -m pytest --timeout=1800 -v tests/firedrake/regression \ -k "poisson_strong or stokes_mini or dg_advection" + # also test for 'problem libraries' (spatialindex and libsupermesh) + python -m pytest -v tests/firedrake/regression/test_locate_cell.py + python -m pytest -v tests/firedrake/supermesh/test_assemble_mixed_mass_matrix.py timeout-minutes: 30 - name: Cleanup (post) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index c358b70141..868c71cf9c 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -56,25 +56,6 @@ jobs: with: path: src/firedrake - - name: Install libsupermesh - run: | - source pip_venv/bin/activate - python -m pip install 'rtree>=1.2' - cd pip_venv/src - git clone https://github.com/firedrakeproject/libsupermesh.git - mkdir -p libsupermesh/build - cd libsupermesh/build - cmake .. \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX="$VIRTUAL_ENV" \ - -DMPI_C_COMPILER="$MPICH_DIR/mpicc" \ - -DMPI_CXX_COMPILER="$MPICH_DIR/mpicxx" \ - -DMPI_Fortran_COMPILER="$MPICH_DIR/mpif90" \ - -DCMAKE_Fortran_COMPILER="$MPICH_DIR/mpif90" \ - -DMPIEXEC_EXECUTABLE="$MPICH_DIR/mpiexec" - make - make install - - name: Pip install run: | source pip_venv/bin/activate diff --git a/.github/workflows/pyop2.yml b/.github/workflows/pyop2.yml index b5e750242a..5104284d05 100644 --- a/.github/workflows/pyop2.yml +++ b/.github/workflows/pyop2.yml @@ -35,17 +35,6 @@ jobs: id: setup-python with: python-version: ${{ matrix.python-version }} - # By default setup-python pollutes the environment in such a way that virtual - # environments cannot be used. This prevents us from building libsupermesh because - # it relies on having rtree installed into a venv. - # https://github.com/actions/setup-python/issues/851 - # https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#using-update-environment-flag - update-environment: false - - - name: Create virtual environment - shell: bash - run: | - ${{ steps.setup-python.outputs.python-path }} -m venv venv - name: Clone PETSc uses: actions/checkout@v4 @@ -65,25 +54,6 @@ jobs: --with-fortran-bindings=0 make - - name: Install libsupermesh - shell: bash - run: | - source venv/bin/activate - python -m pip install 'rtree>=1.2' - git clone https://github.com/firedrakeproject/libsupermesh.git - mkdir -p libsupermesh/build - cd libsupermesh/build - cmake .. \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX="$VIRTUAL_ENV" \ - -DMPI_C_COMPILER=mpicc \ - -DMPI_CXX_COMPILER=mpicxx \ - -DMPI_Fortran_COMPILER=mpif90 \ - -DCMAKE_Fortran_COMPILER=mpif90 \ - -DMPIEXEC_EXECUTABLE=mpiexec - make - make install - - name: Checkout Firedrake uses: actions/checkout@v4 with: @@ -93,7 +63,6 @@ jobs: shell: bash working-directory: firedrake run: | - source ../venv/bin/activate python -m pip install -U pip python -m pip install -U pytest-timeout @@ -101,7 +70,6 @@ jobs: shell: bash working-directory: firedrake run: | - source ../venv/bin/activate export CC=mpicc export HDF5_DIR="$PETSC_DIR/$PETSC_ARCH" export HDF5_MPI=ON @@ -111,7 +79,6 @@ jobs: shell: bash working-directory: firedrake run: | - source ../venv/bin/activate pytest --tb=native --timeout=480 --timeout-method=thread -o faulthandler_timeout=540 -v tests/tsfc timeout-minutes: 10 @@ -119,7 +86,6 @@ jobs: shell: bash working-directory: firedrake run: | - source ../venv/bin/activate pytest -m "not parallel" --tb=native --timeout=480 --timeout-method=thread -o faulthandler_timeout=540 -v tests/pyop2 mpiexec -n 2 --oversubscribe pytest -m "parallel[2]" --tb=native --timeout=480 --timeout-method=thread -o faulthandler_timeout=540 -v tests/pyop2 mpiexec -n 3 --oversubscribe pytest -m "parallel[3]" --tb=native --timeout=480 --timeout-method=thread -o faulthandler_timeout=540 -v tests/pyop2 diff --git a/docker/Dockerfile.complex b/docker/Dockerfile.complex index 1528f5c4ac..6aa73f280c 100644 --- a/docker/Dockerfile.complex +++ b/docker/Dockerfile.complex @@ -18,5 +18,6 @@ RUN bash -c "python3 firedrake-install \ --mpicxx=$MPICH_DIR/mpicxx \ --mpif90=$MPICH_DIR/mpif90 \ --mpiexec=$MPICH_DIR/mpiexec \ + --mpihome=$MPICH_DIR/.. \ --slepc \ --documentation-dependencies" diff --git a/docker/Dockerfile.vanilla b/docker/Dockerfile.vanilla index 2de26b1916..eee3ef53e3 100644 --- a/docker/Dockerfile.vanilla +++ b/docker/Dockerfile.vanilla @@ -16,4 +16,5 @@ RUN bash -c "python3 firedrake-install \ --mpicc=$MPICH_DIR/mpicc \ --mpicxx=$MPICH_DIR/mpicxx \ --mpif90=$MPICH_DIR/mpif90 \ - --mpiexec=$MPICH_DIR/mpiexec" + --mpiexec=$MPICH_DIR/mpiexec \ + --mpihome=$MPICH_DIR/.." diff --git a/docs/source/download.rst b/docs/source/download.rst index 74645fc682..5f8f478864 100644 --- a/docs/source/download.rst +++ b/docs/source/download.rst @@ -241,7 +241,6 @@ Requirements * An activated virtual environment. * All the system requirements listed in :ref:`system-requirements`. * A Firedrake-compatible PETSc installation (using our `fork of PETSc `_). The set of flags passed to PETSc can be retrieved by passing the command ``--show-petsc-configure-options`` to ``firedrake-install``. -* `libsupermesh `_ to be installed inside the virtual environment (see `here `_ for an example of how to do this). * The following environment variables to be set: * ``PETSC_DIR`` and ``PETSC_ARCH`` to point to the correct location for the PETSc installation. diff --git a/docs/source/parallelism.rst b/docs/source/parallelism.rst index 70fc119e98..d71b3d9305 100644 --- a/docs/source/parallelism.rst +++ b/docs/source/parallelism.rst @@ -45,11 +45,12 @@ firedrake installer to use it, by running: .. code-block:: shell - python3 firedrake-install --mpiexec=mpiexec --mpicc=mpicc --mpicxx=mpicxx --mpif90=mpif90 + python3 firedrake-install --mpiexec=mpiexec --mpicc=mpicc --mpicxx=mpicxx --mpif90=mpif90 --mpihome mpihome where ``mpiexec``, ``mpicc``, ``mpicxx``, and ``mpif90`` are the commands to run an MPI job and to compile C, C++, and Fortran 90 code, -respectively. +respectively. ``mpihome`` is an extra variable that must point to the +root directory of the MPI installation (e.g. ``/usr`` or ``/opt/mpich``). Printing in parallel ==================== diff --git a/firedrake/supermeshing.py b/firedrake/supermeshing.py index a1ce2cde17..ee576ea6e5 100644 --- a/firedrake/supermeshing.py +++ b/firedrake/supermeshing.py @@ -1,7 +1,8 @@ # Code for projections and other fun stuff involving supermeshes. import firedrake import ctypes -import sys +import pathlib +import libsupermesh from firedrake.cython.supermeshimpl import assemble_mixed_mass_matrix as ammm, intersection_finder from firedrake.mg.utils import get_level from firedrake.petsc import PETSc @@ -428,7 +429,8 @@ def likely(cell_A): "complex_mode": 1 if complex_mode else 0 } - dirs = get_petsc_dir() + (sys.prefix, ) + libsupermesh_dir = pathlib.Path(libsupermesh.get_include()).parent.absolute() + dirs = get_petsc_dir() + (libsupermesh_dir,) includes = ["-I%s/include" % d for d in dirs] libs = ["-L%s/lib" % d for d in dirs] libs = libs + ["-Wl,-rpath,%s/lib" % d for d in dirs] + ["-lpetsc", "-lsupermesh"] diff --git a/pyproject.toml b/pyproject.toml index fe15d8e7c1..bb934e4182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "fenics-fiat @ git+https://github.com/firedrakeproject/fiat.git", "pyadjoint-ad @ git+https://github.com/dolfin-adjoint/pyadjoint.git", "loopy @ git+https://github.com/firedrakeproject/loopy.git@main", + "libsupermesh @ git+https://github.com/firedrakeproject/libsupermesh.git", ] classifiers = [ "Development Status :: 5 - Production/Stable", @@ -98,6 +99,7 @@ requires = [ "mpi4py; python_version < '3.13'", "petsc4py", "rtree>=1.2", + "libsupermesh @ git+https://github.com/firedrakeproject/libsupermesh.git", ] build-backend = "setuptools.build_meta" diff --git a/requirements-git.txt b/requirements-git.txt index 93e6df0f47..b616f15bbc 100644 --- a/requirements-git.txt +++ b/requirements-git.txt @@ -4,3 +4,4 @@ git+https://github.com/dolfin-adjoint/pyadjoint.git#egg=pyadjoint-ad git+https://github.com/firedrakeproject/loopy.git@main#egg=loopy git+https://github.com/firedrakeproject/pytest-mpi.git@main#egg=pytest-mpi git+https://github.com/firedrakeproject/petsc.git@firedrake#egg=petsc +git+https://github.com/firedrakeproject/libsupermesh.git#egg=libsupermesh diff --git a/scripts/firedrake-install b/scripts/firedrake-install index f4406a6ffb..fa39e2b3b0 100755 --- a/scripts/firedrake-install +++ b/scripts/firedrake-install @@ -76,7 +76,7 @@ class FiredrakeConfiguration(dict): self["options"][o] = args.__dict__[o] _persistent_options = ["package_manager", - "minimal_petsc", "mpicc", "mpicxx", "mpif90", "mpiexec", "disable_ssh", + "minimal_petsc", "mpicc", "mpicxx", "mpif90", "mpiexec", "mpihome", "disable_ssh", "honour_petsc_dir", "with_parmetis", "slepc", "packages", "honour_pythonpath", "opencascade", "torch", "jax", @@ -306,6 +306,9 @@ honoured.""", parser.add_argument("--mpiexec", type=str, action="store", default=None, help="MPI launcher. If not set, MPICH will be downloaded and used.") + parser.add_argument("--mpihome", type=str, + action="store", default=None, + help="Location of MPI files. If not set, MPICH will be downloaded and used.") parser.add_argument("--mpi4py-version", help="Specify an exact version of mpi4py to install") parser.add_argument("--show-petsc-configure-options", action="store_true", help="Print out the configure options passed to PETSc and exit") @@ -339,9 +342,9 @@ honoured.""", args = parser.parse_args() # If the user has set any MPI info, they must set them all - if args.mpicc or args.mpicxx or args.mpif90 or args.mpiexec: - if not (args.mpicc and args.mpicxx and args.mpif90 and args.mpiexec): - log.error("If you set any MPI information, you must set all of {mpicc, mpicxx, mpif90, mpiexec}.") + if args.mpicc or args.mpicxx or args.mpif90 or args.mpiexec or args.mpihome: + if not (args.mpicc and args.mpicxx and args.mpif90 and args.mpiexec and args.mpihome): + log.error("If you set any MPI information, you must set all of {mpicc, mpicxx, mpif90, mpiexec, mpihome}.") sys.exit(1) if args.package_branch: @@ -1252,36 +1255,6 @@ def build_and_install_h5py(): log.info("No need to rebuild h5py") -def build_and_install_libsupermesh(cc, cxx, f90, mpiexec): - log.info("Installing libsupermesh") - url = "git+https://github.com/firedrakeproject/libsupermesh.git" - if os.path.exists("libsupermesh"): - changed = git_update("libsupermesh", url) - else: - git_clone(url) - changed = True - if changed: - with directory("libsupermesh"): - check_call(["git", "reset", "--hard"]) - check_call(["git", "clean", "-f", "-x", "-d"]) - check_call(["mkdir", "-p", "build"]) - with directory("build"): - cmd = [ - "cmake", "..", "-DBUILD_SHARED_LIBS=ON", - "-DCMAKE_INSTALL_PREFIX=" + firedrake_env, - "-DMPI_C_COMPILER=" + cc, - "-DMPI_CXX_COMPILER=" + cxx, - "-DMPI_Fortran_COMPILER=" + f90, - "-DCMAKE_Fortran_COMPILER=" + f90, - "-DMPIEXEC_EXECUTABLE=" + mpiexec, - ] - check_call(cmd) - check_call(["make"]) - check_call(["make", "install"]) - else: - log.info("No need to rebuild libsupermesh") - - def build_and_install_pythonocc(): log.info("Installing pythonocc-core") url = "git+https://github.com/tpaviot/pythonocc-core.git@595b0a4e8e60e8d6011bea0cdb54ac878efcfcd2" @@ -1420,7 +1393,7 @@ if args.rebuild_script: sys.exit(0) -def create_compiler_env(cc, cxx, f90): +def create_compiler_env(cc, cxx, f90, mpihome): env = dict() if cc: env["MPICC"] = cc @@ -1439,8 +1412,10 @@ def create_compiler_env(cc, cxx, f90): env["CXX"] = cxx if f90: env["MPIF90"] = f90 - env["MPI_C_COMPILER"] = f90 + env["MPI_Fortran_COMPILER"] = f90 env["F90"] = f90 + if mpihome: + env["MPI_HOME"] = mpihome return env @@ -1688,6 +1663,8 @@ run_pip(["install", "-U", "hatch"]) run_pip(["install", "-U", "editables"]) run_pip(["install", "-U", "pip"]) run_pip(["install", "-U", "wheel"]) +run_pip(["install", "-U", "scikit-build-core"]) +run_pip(["install", "-U", "rtree>=1.2"]) # Extra numpy dependendencies, see # https://github.com/numpy/numpy/blob/main/pyproject.toml @@ -1733,7 +1710,18 @@ cc = options["mpicc"] cxx = options["mpicxx"] f90 = options["mpif90"] mpiexec = options["mpiexec"] -compiler_env = create_compiler_env(cc, cxx, f90) +try: + mpihome = options["mpihome"] +except KeyError: + if cc: + try: + mpihome = os.environ["MPI_HOME"] + except KeyError: + raise InstallError("MPI compilers specified but mpihome is missing from " + "configuration. Please set MPI_HOME to continue.") + else: + mpihome = None +compiler_env = create_compiler_env(cc, cxx, f90, mpihome) os.chdir(firedrake_env) # Dict to store BLAS and OPENBLAS environment variables @@ -1830,7 +1818,8 @@ if mode == "install": cxx = os.path.join(compilerbin, "mpicxx") f90 = os.path.join(compilerbin, "mpif90") mpiexec = os.path.join(compilerbin, "mpiexec") - compiler_env = create_compiler_env(cc, cxx, f90) + mpihome = os.path.join(compilerbin, "..") + compiler_env = create_compiler_env(cc, cxx, f90, mpihome) # Make sure that we link against the right MPI and PETSc shared libraries link_env = { @@ -1877,9 +1866,6 @@ if mode == "install": install(p+"/") sys.path.append(os.getcwd() + "/" + p) - with environment(**compiler_env): - build_and_install_libsupermesh(cc, cxx, f90, mpiexec) - with pipargs("--no-deps"), environment(**compiler_env, **link_env): install("firedrake/") @@ -1986,7 +1972,8 @@ else: cxx = os.path.join(compilerbin, "mpicxx") f90 = os.path.join(compilerbin, "mpif90") mpiexec = os.path.join(compilerbin, "mpiexec") - compiler_env = create_compiler_env(cc, cxx, f90) + mpihome = os.path.join(compilerbin, "..") + compiler_env = create_compiler_env(cc, cxx, f90, mpihome) # Make sure that we link against the right MPI and PETSc shared libraries link_env = { @@ -2059,9 +2046,6 @@ Please consider updating your PETSc manually. else: install(p+"/") - with environment(**compiler_env): - build_and_install_libsupermesh(cc, cxx, f90, mpiexec) - with pipargs("--no-deps"), environment(**compiler_env, **link_env): install("firedrake/") diff --git a/setup.py b/setup.py index cf970c1712..52112e1500 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import pybind11 import petsc4py import rtree +import libsupermesh import pkgconfig from dataclasses import dataclass, field from setuptools import setup, find_packages, Extension @@ -145,6 +146,9 @@ def __getitem__(self, key): # In the next 2 linkages we are using `site.getsitepackages()[0]`, which isn't # guaranteed to be the correct place we could also use "$ORIGIN/../../lib_dir", # but that definitely doesn't work with editable installs. +# This is necessary because Python build isolation means that the compile-time +# library dirs (in the isolated build env) are different to the run-time +# library dirs (in the venv). # libspatialindex # example: @@ -159,15 +163,15 @@ def __getitem__(self, key): # libsupermesh # example: -# gcc -I/supermesh/include -# gcc /supermesh/supermesh.cpython-311-x86_64-linux-gnu.so \ +# gcc -Ipath/to/libsupermesh/include +# gcc path/to/libsupermesh/libsupermesh.cpython-311-x86_64-linux-gnu.so \ # -lsupermesh \ -# -Wl,-rpath,$ORIGIN/../../supermesh -supermesh_ = ExternalDependency( - include_dirs=[f"{sys.prefix}/include"], - library_dirs=[f"{sys.prefix}/lib"], +# -Wl,-rpath,$ORIGIN/../../libsupermesh +libsupermesh_ = ExternalDependency( + include_dirs=[libsupermesh.get_include()], + library_dirs=[str(Path(libsupermesh.get_library()).parent)], + runtime_library_dirs=[os.path.join(site.getsitepackages()[0], "libsupermesh", "lib")], libraries=["supermesh"], - runtime_library_dirs=[f"{sys.prefix}/lib"], ) # The following extensions need to be linked accordingly: @@ -221,7 +225,7 @@ def extensions(): name="firedrake.cython.supermeshimpl", language="c", sources=[os.path.join("firedrake", "cython", "supermeshimpl.pyx")], - **(petsc_ + numpy_ + supermesh_) + **(petsc_ + numpy_ + libsupermesh_) )) # pyop2/sparsity.pyx: petsc, numpy, cython_list.append(Extension(