From ab60937b1d19457d2fa8eb23dffd10cc9934577a Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Mon, 8 May 2023 10:37:01 +0200 Subject: [PATCH 1/5] plumed: allow for ILP64 builds --- .../libraries/science/chemistry/plumed/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/science/chemistry/plumed/default.nix b/pkgs/development/libraries/science/chemistry/plumed/default.nix index b825c8270d628..2d1388b3008a6 100644 --- a/pkgs/development/libraries/science/chemistry/plumed/default.nix +++ b/pkgs/development/libraries/science/chemistry/plumed/default.nix @@ -4,8 +4,6 @@ , blas }: -assert !blas.isILP64; - stdenv.mkDerivation rec { pname = "plumed"; version = "2.9.0"; @@ -21,6 +19,8 @@ stdenv.mkDerivation rec { patchShebangs . ''; + configureFlags = [] ++ lib.optional blas.isILP64 "--enable-ilp64"; + buildInputs = [ blas ]; enableParallelBuilding = true; From f36394c6164a71f72d3d98159b48ec5ba6861f16 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Tue, 2 May 2023 12:33:40 +0200 Subject: [PATCH 2/5] prrte: init at 3.0.3 --- pkgs/by-name/pr/prrte/package.nix | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 pkgs/by-name/pr/prrte/package.nix diff --git a/pkgs/by-name/pr/prrte/package.nix b/pkgs/by-name/pr/prrte/package.nix new file mode 100644 index 0000000000000..8eec46bdb1c31 --- /dev/null +++ b/pkgs/by-name/pr/prrte/package.nix @@ -0,0 +1,61 @@ +{ lib +, stdenv +, removeReferencesTo +, fetchFromGitHub +, autoconf +, automake +, libtool +, gitMinimal +, perl +, python3 +, flex +, hwloc +, libevent +, zlib +, pmix +}: + +stdenv.mkDerivation rec { + pname = "prrte"; + version = "3.0.5"; + + src = fetchFromGitHub { + owner = "openpmix"; + repo = "prrte"; + rev = "v${version}"; + sha256 = "sha256-RDxd4veLGbN+T7xCDnNp2lbOM7mwKKD+SKdPmExr1C8="; + fetchSubmodules = true; + }; + + outputs = [ "out" "dev" ]; + + postPatch = '' + patchShebangs ./autogen.pl + patchShebangs ./config + ''; + + preConfigure = '' + ./autogen.pl + ''; + + postInstall = '' + moveToOutput "bin/prte_info" "''${!outputDev}" + moveToOutput "bin/pcc" "''${!outputDev}" + + remove-references-to -t $dev $(readlink -f $out/lib/libprrte${stdenv.hostPlatform.extensions.library}) + ''; + + nativeBuildInputs = [ removeReferencesTo perl python3 autoconf automake libtool flex gitMinimal ]; + + buildInputs = [ libevent hwloc zlib pmix ]; + + enableParallelBuilding = true; + + meta = with lib; { + description = "PMIx Reference Runtime Environment"; + homepage = "https://docs.prrte.org/"; + license = licenses.bsd3; + maintainers = [ maintainers.markuskowa ]; + platforms = platforms.linux; + }; +} From c2e58ed8de03d1146267299445ccd5c9d8fe0a6e Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Tue, 30 Apr 2024 16:09:02 +0200 Subject: [PATCH 3/5] pythonPackages.mpi4py: disable flaky test --- pkgs/development/python-modules/mpi4py/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/development/python-modules/mpi4py/default.nix b/pkgs/development/python-modules/mpi4py/default.nix index 43c4040fba72d..ea3efdcf105c5 100644 --- a/pkgs/development/python-modules/mpi4py/default.nix +++ b/pkgs/development/python-modules/mpi4py/default.nix @@ -26,6 +26,10 @@ buildPythonPackage rec { substituteInPlace test/test_spawn.py --replace \ "unittest.skipMPI('openmpi(<3.0.0)')" \ "unittest.skipMPI('openmpi')" + + substituteInPlace test/test_dynproc.py --replace \ + "unittest.skipMPI('openmpi(<2.0.0)')" \ + "unittest.skipMPI('openmpi')" \ ''; configurePhase = ""; From 5c43fdd254ac6b8c443be11a5554b4e92d3e2d95 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Sat, 29 Apr 2023 10:46:14 +0200 Subject: [PATCH 4/5] openmpi: 4.1.6 -> 5.0.3 * add prrte * cleanup reproducible builds; fix compiler wrappers * fix detection/selection of AVX/SSE Patch out automatic detection for more reproducible builds At least AVX needs to enabled to build. --- .../development/libraries/openmpi/default.nix | 99 +++++++++++-------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix index 8c54c13c9139e..08195b6cb5517 100644 --- a/pkgs/development/libraries/openmpi/default.nix +++ b/pkgs/development/libraries/openmpi/default.nix @@ -1,7 +1,8 @@ { lib, stdenv, fetchurl, removeReferencesTo, gfortran, perl, libnl , rdma-core, zlib, numactl, libevent, hwloc, targetPackages, symlinkJoin -, libpsm2, libfabric, pmix, ucx, ucc, makeWrapper +, libpsm2, libfabric, pmix, ucx, ucc, prrte, makeWrapper , config + # Enable CUDA support , cudaSupport ? config.cudaSupport, cudaPackages @@ -16,34 +17,56 @@ # Enable Fortran support , fortranSupport ? true + +# AVX/SSE options +# note that opempi fails to build with AVX disabled. +# => everything up to AVX is enabled by default +, enableSse3 ? true +, enableSse4_1 ? true +, enableAvx ? true +, enableAvx2 ? stdenv.hostPlatform.avx2Support +, enableAvx512 ? stdenv.hostPlatform.avx512Support }: stdenv.mkDerivation rec { pname = "openmpi"; - version = "4.1.6"; + version = "5.0.3"; src = with lib.versions; fetchurl { url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2"; - sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU="; + sha256 = "sha256-mQWC8gazqzLpOKoxu/B8Y5No5EBdyhlvq+fw927tqQs="; }; postPatch = '' patchShebangs ./ - # Ensure build is reproducible - ts=`date -d @$SOURCE_DATE_EPOCH` - sed -i 's/OPAL_CONFIGURE_USER=.*/OPAL_CONFIGURE_USER="nixbld"/' configure - sed -i 's/OPAL_CONFIGURE_HOST=.*/OPAL_CONFIGURE_HOST="localhost"/' configure - sed -i "s/OPAL_CONFIGURE_DATE=.*/OPAL_CONFIGURE_DATE=\"$ts\"/" configure - find -name "Makefile.in" -exec sed -i "s/\`date\`/$ts/" \{} \; + # This is dynamically detected. Configure does not provide fine grained options + # We just disable the check in the configure script for now + ${lib.optionalString (!enableSse3) + "substituteInPlace configure --replace-fail 'ompi_cv_op_avx_check_sse3=yes' 'ompi_cv_op_avx_check_sse3=no'"} + ${lib.optionalString (!enableSse4_1) + "substituteInPlace configure --replace-fail 'ompi_cv_op_avx_check_sse41=yes' 'ompi_cv_op_avx_check_sse41=no'"} + ${lib.optionalString (!enableAvx) + "substituteInPlace configure --replace-fail 'ompi_cv_op_avx_check_avx=yes' 'ompi_cv_op_avx_check_avx=no'"} + ${lib.optionalString (!enableAvx2) + "substituteInPlace configure --replace-fail 'ompi_cv_op_avx_check_avx2=yes' 'ompi_cv_op_avx_check_avx2=no'"} + ${lib.optionalString (!enableAvx512) + "substituteInPlace configure --replace-fail 'ompi_cv_op_avx_check_avx512=yes' 'ompi_cv_op_avx_check_av512=no'"} + ''; + + preConfigure = '' + # Ensure build is reproducible according to manual + # https://docs.open-mpi.org/en/v5.0.x/release-notes/general.html#general-notes + export USER=nixbld + export HOSTNAME=localhost + export SOURCE_DATE_EPOCH=0 ''; outputs = [ "out" "man" "dev" ]; - buildInputs = [ zlib ] - ++ lib.optionals stdenv.isLinux [ libnl numactl pmix ucx ucc ] + buildInputs = [ zlib libevent hwloc ] + ++ lib.optionals stdenv.isLinux [ numactl pmix ucx ucc prrte ] ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ] - ++ [ libevent hwloc ] ++ lib.optional (stdenv.isLinux || stdenv.isFreeBSD) rdma-core ++ lib.optionals fabricSupport [ libpsm2 libfabric ]; @@ -52,26 +75,26 @@ stdenv.mkDerivation rec { ++ lib.optionals fortranSupport [ gfortran ]; configureFlags = lib.optional (!cudaSupport) "--disable-mca-dso" + ++ lib.optional stdenv.isLinux "--with-prrte=${lib.getBin prrte}" ++ lib.optional (!fortranSupport) "--disable-mpi-fortran" - ++ lib.optionals stdenv.isLinux [ - "--with-libnl=${lib.getDev libnl}" - "--with-pmix=${lib.getDev pmix}" - "--with-pmix-libdir=${pmix}/lib" - "--enable-mpi-cxx" - ] ++ lib.optional enableSGE "--with-sge" + ++ lib.optional enableSGE "--with-sge" ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default" # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build # https://github.com/openucx/ucx # https://www.open-mpi.org/faq/?category=buildcuda ++ lib.optionals cudaSupport [ "--with-cuda=${cudaPackages.cuda_cudart}" "--enable-dlopen" ] - ++ lib.optionals fabricSupport [ "--with-psm2=${lib.getDev libpsm2}" "--with-libfabric=${lib.getDev libfabric}" ] - ; + ++ lib.optionals fabricSupport [ + "--with-psm2=${lib.getDev libpsm2}" + "--with-libfabric=${lib.getDev libfabric}" + "--with-libfabric-libdir=${lib.getLib libfabric}/lib" + ]; enableParallelBuilding = true; postInstall = '' find $out/lib/ -name "*.la" -exec rm -f \{} \; + # Move compiler wrappers to $dev for f in mpi shmem osh; do for i in f77 f90 CC c++ cxx cc fort; do moveToOutput "bin/$f$i" "''${!outputDev}" @@ -88,34 +111,28 @@ stdenv.mkDerivation rec { ''; postFixup = '' - remove-references-to -t $dev $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary}) - remove-references-to -t $man $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary}) + # These reference are caused by hard coded "configure options" + remove-references-to -t $dev $out/bin/mpirun + remove-references-to -t $dev $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.library}) + remove-references-to -t $man $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.library}) # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared. wrapProgram $dev/bin/opal_wrapper \ --set OPAL_INCLUDEDIR $dev/include \ --set OPAL_PKGDATADIR $dev/share/openmpi - # default compilers should be indentical to the - # compilers at build time - - echo "$dev/share/openmpi/mpicc-wrapper-data.txt" - sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ - $dev/share/openmpi/mpicc-wrapper-data.txt - - echo "$dev/share/openmpi/ortecc-wrapper-data.txt" - sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ - $dev/share/openmpi/ortecc-wrapper-data.txt - - echo "$dev/share/openmpi/mpic++-wrapper-data.txt" - sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \ - $dev/share/openmpi/mpic++-wrapper-data.txt + # default compilers should be identical to the compilers at build time + for api in mpi shmem; do + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ + $dev/share/openmpi/''${api}cc-wrapper-data.txt + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \ + $dev/share/openmpi/''${api}c++-wrapper-data.txt + done '' + lib.optionalString fortranSupport '' - - echo "$dev/share/openmpi/mpifort-wrapper-data.txt" - sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ - $dev/share/openmpi/mpifort-wrapper-data.txt - + for api in mpi shmem; do + sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ + $dev/share/openmpi/''${api}fort-wrapper-data.txt + done ''; doCheck = true; From 2b1202279df94d7d90fa81ffae555af48af525b5 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Wed, 5 Jun 2024 10:56:58 +0200 Subject: [PATCH 5/5] openmpi: disable pmix and split output on non-Linux --- .../development/libraries/openmpi/default.nix | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix index 08195b6cb5517..4cfed6d18046e 100644 --- a/pkgs/development/libraries/openmpi/default.nix +++ b/pkgs/development/libraries/openmpi/default.nix @@ -1,6 +1,6 @@ { lib, stdenv, fetchurl, removeReferencesTo, gfortran, perl, libnl , rdma-core, zlib, numactl, libevent, hwloc, targetPackages, symlinkJoin -, libpsm2, libfabric, pmix, ucx, ucc, prrte, makeWrapper +, libpsm2, libfabric, pmix, ucx, ucc, prrte, makeWrapper, python3 , config # Enable CUDA support @@ -62,13 +62,15 @@ stdenv.mkDerivation rec { export SOURCE_DATE_EPOCH=0 ''; - outputs = [ "out" "man" "dev" ]; + # Split output is too fragile. Limit to Linux. + outputs = if stdenv.isLinux then [ "out" "man" "dev" ] else [ "out" ]; buildInputs = [ zlib libevent hwloc ] ++ lib.optionals stdenv.isLinux [ numactl pmix ucx ucc prrte ] ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ] ++ lib.optional (stdenv.isLinux || stdenv.isFreeBSD) rdma-core - ++ lib.optionals fabricSupport [ libpsm2 libfabric ]; + ++ lib.optionals fabricSupport [ libpsm2 libfabric ] + ++ lib.optional (!stdenv.isLinux) python3; # needed for internal pmix nativeBuildInputs = [ perl removeReferencesTo makeWrapper ] ++ lib.optionals cudaSupport [ cudaPackages.cuda_nvcc ] @@ -87,11 +89,12 @@ stdenv.mkDerivation rec { "--with-psm2=${lib.getDev libpsm2}" "--with-libfabric=${lib.getDev libfabric}" "--with-libfabric-libdir=${lib.getLib libfabric}/lib" - ]; + ] + ++ lib.optional (!stdenv.isLinux) "--with-pmix=internal"; enableParallelBuilding = true; - postInstall = '' + postInstall = lib.optionalString stdenv.isLinux '' find $out/lib/ -name "*.la" -exec rm -f \{} \; # Move compiler wrappers to $dev @@ -110,7 +113,7 @@ stdenv.mkDerivation rec { moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}" ''; - postFixup = '' + postFixup = lib.optionalString stdenv.isLinux '' # These reference are caused by hard coded "configure options" remove-references-to -t $dev $out/bin/mpirun remove-references-to -t $dev $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.library}) @@ -123,15 +126,21 @@ stdenv.mkDerivation rec { # default compilers should be identical to the compilers at build time for api in mpi shmem; do - sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ - $dev/share/openmpi/''${api}cc-wrapper-data.txt - sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \ - $dev/share/openmpi/''${api}c++-wrapper-data.txt + if [ -e $dev/share/openmpi/''${api}cc-wrapper-data.txt ]; then + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ + $dev/share/openmpi/''${api}cc-wrapper-data.txt + fi + if [ -e $dev/share/openmpi/''${api}c++-wrapper-data.txt ]; then + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \ + $dev/share/openmpi/''${api}c++-wrapper-data.txt + fi done - '' + lib.optionalString fortranSupport '' + '' + lib.optionalString (fortranSupport && stdenv.isLinux) '' for api in mpi shmem; do - sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ - $dev/share/openmpi/''${api}fort-wrapper-data.txt + if [ -e $dev/share/openmpi/''${api}fort-wrapper-data.txt ]; then + sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ + $dev/share/openmpi/''${api}fort-wrapper-data.txt + fi done '';