CP2K 2024.3 and F7T runner for Todi (#148)

Users hit a bug in CP2K affecting MD and cell relaxation, that has been fixed in 2024.3.

cp2k/cp2k#3661

config.yaml

@@ -16,16 +16,8 @@ clusters:
  uarch: 'gh200'
  partition: 'normal'
  variables:
- # new daint has a custom url (temporary?)
- F7T_URL: "https://firecrest-hpc.v1.svc.cscs.ch"
+ F7T_URL: "https://api.cscs.ch/hpc/firecrest/v1"
  runner: f7t
- #targets:
- # -
- # uarch: 'gh200'
- # partition: 'normal'
- #runner:
- # slurm-tag: daint-spack-stack-builder
- # baremetal-tag: daint-login-baremetal
  eiger:
  targets:
  -
@@ -57,9 +49,9 @@ clusters:
  -
  uarch: 'gh200'
  partition: 'normal'
- runner:
- slurm-tag: todi-spack-stack-builder
-  baremetal-tag: todi-login-baremetal
+  variables:
+  F7T_URL: "https://api.cscs.ch/cscs/firecrest/v1"
+ runner: f7t
 uenvs:
  arbor:
  "v0.9":
@@ -120,6 +112,10 @@ uenvs:
  recipes:
  gh200: 2024.2/gh200
  zen2: 2024.2/mc
+ "2024.3":
+ recipes:
+ gh200: 2024.3/gh200
+ zen2: 2024.3/mc
  deploy:
  daint: [gh200]
  todi: [gh200]

recipes/cp2k/2024.3/gh200/compilers.yaml

@@ -0,0 +1,5 @@
+bootstrap:
+ spec: gcc@12
+gcc:
+ specs:
+ - [email protected]

recipes/cp2k/2024.3/gh200/config.yaml

@@ -0,0 +1,6 @@
+name: cp2k
+store: /user-environment
+spack:
+ repo: https://github.com/spack/spack.git
+ commit: v0.22.1
+modules: True

recipes/cp2k/2024.3/gh200/environments.yaml

@@ -0,0 +1,44 @@
+cp2k:
+ compiler:
+ - toolchain: gcc
+ spec: [email protected]
+ unify: true
+ specs:
+ - cmake
+ - ninja
+ - [email protected] # Can't be newer than driver version, for DBCSR
+ - openblas threads=openmp
+ - netlib-scalapack
+ - fftw
+ - libxc ~cuda
+ - libint
+ - spglib
+ - spla
+ - elpa
+ - plumed
+ # COSMA
+ - tiled-mm
+ - costa
+ - cosma +gpu_direct
+ # SIRIUS
+ - hdf5
+ - spfft
+ - sirius ~memory_pool
+ # CP2K
+ - [email protected]
+ - [email protected] +libxc +libint +spglib +cosma +spla +elpa +sirius +plumed lmax=5 build_system=cmake
+ variants:
+ - +mpi
+ - +openmp
+ - +cuda
+ - cuda_arch=90
+ - build_type=Release
+ mpi:
+ spec: [email protected]
+ gpu: cuda
+ views:
+ develop:
+ link: roots
+ exclude: ["cp2k"]
+ cp2k:
+ link: roots

recipes/cp2k/2024.3/gh200/modules.yaml

@@ -0,0 +1,21 @@
+modules:
+ prefix_inspections:
+ bin:
+ - PATH
+ lib:
+ - LD_LIBRARY_PATH
+ lib64:
+ - LD_LIBRARY_PATH
+
+ default:
+ arch_folder: false
+ roots:
+ tcl: /user-environment/modules
+ tcl:
+ all:
+ autoload: none
+ hash_length: 0
+ exclude_implicits: true
+ exclude: ['%[email protected]', 'gcc %[email protected]']
+ projections:
+ all: '{name}/{version}'

recipes/cp2k/2024.3/gh200/repo/packages/cosma/fj-ssl2.patch

@@ -0,0 +1,104 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 1fd1e55..41a041b 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -19,7 +19,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS "YES") # always write compile_commands.json
+
+ set(COSMA_GPU_BACKENDS_LIST "CUDA" "ROCM")
+ set(COSMA_SCALAPACK_LIST "OFF" "MKL" "CRAY_LIBSCI" "CUSTOM")
+-set(COSMA_BLAS_LIST "auto" "MKL" "OPENBLAS" "CRAY_LIBSCI" "CUSTOM" "BLIS" "ATLAS" "CUDA" "ROCM" "OFF")
++set(COSMA_BLAS_LIST "auto" "MKL" "SSL2" "OPENBLAS" "CRAY_LIBSCI" "CUSTOM" "BLIS" "ATLAS" "CUDA" "ROCM" "OFF")
+ option(COSMA_WITH_TESTS "Generate the test target." ON)
+ option(COSMA_WITH_APPS "Generate the miniapp targets." ON)
+ option(COSMA_WITH_BENCHMARKS "Generate the benchmark targets." ON)
+@@ -45,7 +45,7 @@ if (COSMA_BLAS MATCHES "CUDA|ROCM")
+ set(COSMA_GPU_BACKEND ${COSMA_BLAS})
+ else()
+ if(COSMA_BLAS STREQUAL "OFF")
+- message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, OPENBLAS, CRAY_LIBSCI, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
++ message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, SSL2, OPENBLAS, CRAY_LIBSCI, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
+ else()
+ set(COSMA_BLAS_VENDOR ${COSMA_BLAS})
+ endif()
+@@ -190,6 +190,7 @@ install(FILES "${cosma_BINARY_DIR}/cosmaConfig.cmake"
+ "${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
+ "${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
+ "${cosma_SOURCE_DIR}/cmake/FindMKL.cmake"
++ "${cosma_SOURCE_DIR}/cmake/FindSSL2.cmake"
+ "${cosma_SOURCE_DIR}/cmake/FindBlas.cmake"
+ "${cosma_SOURCE_DIR}/cmake/FindSCALAPACK.cmake"
+ "${cosma_SOURCE_DIR}/cmake/FindOPENBLAS.cmake"
+diff --git a/cmake/FindBlas.cmake b/cmake/FindBlas.cmake
+index aef956c..3c47561 100644
+--- a/cmake/FindBlas.cmake
++++ b/cmake/FindBlas.cmake
+@@ -14,6 +14,7 @@ endif()
+ set(COSMA_BLAS_VENDOR_LIST
+ "auto"
+ "MKL"
++ "SSL2"
+ "OPENBLAS"
+ "FLEXIBLAS"
+ "ARMPL"
+diff --git a/cmake/FindSSL2.cmake b/cmake/FindSSL2.cmake
+new file mode 100644
+index 0000000..f0e11bf
+--- /dev/null
++++ b/cmake/FindSSL2.cmake
+@@ -0,0 +1,56 @@
++#.rst:
++# FindSSL2
++# -----------
++#
++# This module tries to find the SSL2 library.
++#
++# The following variables are set
++#
++# ::
++#
++# SSL2_FOUND - True if ssl2 is found
++# SSL2_LIBRARIES - The required libraries
++# SSL2_INCLUDE_DIRS - The required include directory
++#
++# The following import target is created
++#
++# ::
++#
++# SSL2::ssl2
++
++#set paths to look for library from ROOT variables.If new policy is set, find_library() automatically uses them.
++# if(NOT POLICY CMP0074)
++set(_SSL2_PATHS ${SSL2_ROOT}
++ $ENV{SSL2_ROOT}
++ $ENV{SSL2ROOT}
++ $ENV{SSL2_DIR}
++ $ENV{SSL2DIR})
++# endif()
++
++find_library(
++ COSMA_SSL2_LINK_LIBRARIES
++ NAMES "fjlapackex"
++ HINTS ${_SSL2_PATHS}
++ PATH_SUFFIXES "lib64"
++)
++find_path(
++ COSMA_SSL2_INCLUDE_DIRS
++ NAMES "cblas.h" 
++ HINTS ${_SSL2_PATHS}
++ PATH_SUFFIXES "include"
++)
++
++# check if found
++include(FindPackageHandleStandardArgs)
++find_package_handle_standard_args(SSL2 REQUIRED_VARS COSMA_SSL2_INCLUDE_DIRS COSMA_SSL2_LINK_LIBRARIES)
++
++# add target to link against
++if(NOT TARGET cosma::BLAS::SSL2::ssl2)
++ add_library(cosma::BLAS::SSL2::ssl2 INTERFACE IMPORTED)
++ add_library(cosma::BLAS::SSL2::blas ALIAS cosma::BLAS::SSL2::ssl2)
++endif()
++set_property(TARGET cosma::BLAS::SSL2::ssl2 PROPERTY INTERFACE_LINK_LIBRARIES ${COSMA_SSL2_LINK_LIBRARIES})
++set_property(TARGET cosma::BLAS::SSL2::ssl2 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${COSMA_SSL2_INCLUDE_DIRS})
++
++# prevent clutter in cache
++MARK_AS_ADVANCED(SSL2_FOUND SSL2_LIBRARIES SSL2_INCLUDE_DIRS)

recipes/cp2k/2024.3/gh200/repo/packages/cosma/mpi-view.patch

@@ -0,0 +1,13 @@
+diff --git a/cmake/cosmaConfig.cmake.in b/cmake/cosmaConfig.cmake.in
+index cf7038f..1527c53 100644
+--- a/cmake/cosmaConfig.cmake.in
++++ b/cmake/cosmaConfig.cmake.in
+@@ -11,7 +11,7 @@ if(NOT TARGET cosma::cosma)
+ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_MODULE_PATH})
+
+ set(MKL_ROOT "@MKL_ROOT@")
+- set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
++ # set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
+ set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+ find_package(MPI COMPONENTS "CXX" REQUIRED)
+

recipes/cp2k/2024.3/gh200/repo/packages/cosma/package.py

@@ -0,0 +1,135 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Cosma(CMakePackage):
+ """
+ Distributed Communication-Optimal Matrix-Matrix Multiplication Library
+ """
+
+ maintainers("haampie", "kabicm", "teonnik", "simonpintarelli", "mtaillefumier")
+ homepage = "https://github.com/eth-cscs/COSMA"
+ url = "https://github.com/eth-cscs/COSMA/archive/refs/tags/v2.6.6.tar.gz"
+ git = "https://github.com/eth-cscs/COSMA.git"
+
+ license("BSD-3-Clause")
+
+ # note: The default archives produced with github do not have the archives
+ # of the submodules.
+ version("master", branch="master", submodules=False)
+ version("2.6.6", sha256="1604be101e77192fbcc5551236bc87888d336e402f5409bbdd9dea900401cc37")
+ version("2.6.5", sha256="10d9b7ecc1ce44ec5b9e0c0bf89278a63029912ec3ea99661be8576b553ececf")
+ version("2.6.4", sha256="6d7bd5e3005874af9542a329c93e7ccd29ca1a5573dae27618fac2704fa2b6ab")
+ version("2.6.3", sha256="c2a3735ea8f860930bea6706d968497d72a1be0498c689b5bc4a951ffc2d1146")
+ version("2.6.2", sha256="2debb5123cc35aeebc5fd2f8a46cfd6356d1e27618c9bb57129ecd09aa400940")
+ version("2.6.1", sha256="69aa6634a030674f0d9be61e7b0bf0dc17acf0fc9e7a90b40e3179e2254c8d67")
+ version("2.5.1", sha256="085b7787597374244bbb1eb89bc69bf58c35f6c85be805e881e1c0b25166c3ce")
+ version("2.5.0", sha256="7f68bb0ee5c80f9b8df858afcbd017ad4ed87ac09439d13d7d890844dbdd3d54")
+ version("2.4.0", sha256="5714315ce06d48037f86cfee2d7f19340643fee95e9d7f1e92dc1b623b67e395")
+ version("2.3.0", sha256="0c01c2deb5a0cd177952178350188a62c42ce55e604d7948ac472f55bf0d4815")
+ version("2.2.0", sha256="1eb92a98110df595070a12193b9221eecf9d103ced8836c960f6c79a2bd553ca")
+ version("2.0.7", sha256="8d70bfcbda6239b6a8fbeaca138790bbe58c0c3aa576879480d2632d4936cf7e")
+ version("2.0.2", sha256="4f3354828bc718f3eef2f0098c3bdca3499297497a220da32db1acd57920c68d")
+
+ # We just need the libraries of cuda and rocm, so no need to extend
+ # CudaPackage or ROCmPackage.
+ variant("cuda", default=False, description="Build with cuBLAS support")
+ variant("rocm", default=False, description="Build with rocBLAS support")
+ variant("scalapack", default=False, description="Build with ScaLAPACK API")
+ variant("shared", default=True, description="Build the shared library version")
+ variant("tests", default=False, description="Build tests")
+ variant("apps", default=False, description="Build miniapp")
+ variant("profiling", default=False, description="Enable profiling")
+ variant("gpu_direct", default=False, description="GPU aware MPI")
+
+ with when("+cuda"):
+ variant("nccl", default=False, description="Use cuda nccl")
+
+ with when("+rocm"):
+ variant("rccl", default=False, description="Use rocm rccl")
+
+ depends_on("[email protected]:", type="build")
+ depends_on("mpi@3:")
+ depends_on("blas", when="~cuda ~rocm")
+ depends_on("scalapack", when="+scalapack")
+ depends_on("cuda", when="+cuda")
+ depends_on("rocblas", when="+rocm")
+ depends_on("nccl", when="+nccl")
+ depends_on("rccl", when="+rccl")
+
+ with when("@2.6.3:"):
+ depends_on("[email protected]:+cuda", when="+cuda")
+ depends_on("[email protected]:+rocm", when="+rocm")
+
+ with when("@2.6.1:2.6.2"):
+ depends_on("[email protected]+rocm", when="+rocm")
+ depends_on("[email protected]+cuda", when="+cuda")
+
+ with when("@2.6.1:"):
+ depends_on("costa")
+ depends_on("costa+scalapack", when="+scalapack")
+ depends_on("cxxopts", when="+apps")
+ depends_on("cxxopts", when="+tests")
+ depends_on("semiprof", when="+profiling")
+ depends_on("costa+profiling", when="+profiling")
+
+ patch("fj-ssl2.patch", when="^fujitsu-ssl2")
+ patch("mpi-view.patch")
+
+ def setup_build_environment(self, env):
+ if "+cuda" in self.spec:
+ env.set("CUDA_PATH", self.spec["cuda"].prefix)
+
+ def cosma_blas_cmake_arg(self):
+ query_to_cmake_arg = [
+ ("+cuda", "CUDA"),
+ ("+rocm", "ROCM"),
+ ("^intel-mkl", "MKL"),
+ ("^intel-oneapi-mkl", "MKL"),
+ ("^cray-libsci", "CRAY_LIBSCI"),
+ ("^netlib-lapack", "CUSTOM"),
+ ("^openblas", "OPENBLAS"),
+ ("^fujitsu-ssl2", "SSL2"),
+ ]
+
+ if self.version >= Version("2.4.0"):
+ query_to_cmake_arg.extend(
+ [("^blis", "BLIS"), ("^amdblis", "BLIS"), ("^atlas", "ATLAS")]
+ )
+
+ for query, cmake_arg in query_to_cmake_arg:
+ if query in self.spec:
+ return cmake_arg
+
+ return "CUSTOM"
+
+ def cosma_scalapack_cmake_arg(self):
+ spec = self.spec
+
+ if "~scalapack" in spec:
+ return "OFF"
+ elif "^intel-mkl" in spec or "^intel-oneapi-mkl" in spec:
+ return "MKL"
+ elif "^cray-libsci" in spec:
+ return "CRAY_LIBSCI"
+
+ return "CUSTOM"
+
+ def cmake_args(self):
+ return [
+ self.define_from_variant("COSMA_WITH_TESTS", "tests"),
+ self.define_from_variant("COSMA_WITH_APPS", "apps"),
+ self.define_from_variant("COSMA_WITH_NCCL", "nccl"),
+ self.define_from_variant("COSMA_WITH_RCCL", "rccl"),
+ self.define_from_variant("COSMA_WITH_GPU_AWARE_MPI", "gpu_direct"),
+ self.define_from_variant("COSMA_WITH_PROFILING", "profiling"),
+ self.define("COSMA_WITH_BENCHMARKS", False),
+ self.define("COSMA_BLAS", self.cosma_blas_cmake_arg()),
+ self.define("COSMA_SCALAPACK", self.cosma_scalapack_cmake_arg()),
+ self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+ ]

recipes/cp2k/2024.3/gh200/repo/packages/costa/mpi-view.patch

@@ -0,0 +1,13 @@
+diff --git a/cmake/costaConfig.cmake.in b/cmake/costaConfig.cmake.in
+index 8fae534..d744c90 100644
+--- a/cmake/costaConfig.cmake.in
++++ b/cmake/costaConfig.cmake.in
+@@ -10,7 +10,7 @@ include(CMakeFindDependencyMacro)
+ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_MODULE_PATH})
+
+ set(MKL_ROOT "@MKL_ROOT@")
+-set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
++#set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
+
+ set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+ set(COSTA_SCALAPACK "@COSTA_SCALAPACK@")