Merge branch 'dev' of https://github.com/fmihpc/vlasiator into dev

.github/workflows/github-ci.yml

@@ -22,13 +22,13 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  - name: Run library build script
  run: ./build_libraries.sh
  - name: Build libraries tar
  run: tar --zstd -cvf libraries.tar.zstd libraries/
  - name: Upload libraries as artifact
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
  name: libraries
  path: libraries.tar.zstd
@@ -41,7 +41,7 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  - name: Setup MPI
  uses: mpi4py/setup-mpi@v1
  with:
@@ -61,7 +61,7 @@ jobs:
  - name: Build libraries tar
  run: tar --zstd -cvf libraries-appleM1.tar.zstd libraries-appleM1
  - name: Upload libraries as artifact
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
  name: libraries-appleM1
  path: libraries-appleM1.tar.zstd
@@ -75,7 +75,7 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - name: Clean workspace
@@ -88,7 +88,7 @@ jobs:
  - name: Build libraries tar
  run: tar -czvf libraries-arriesgado.tar.gz libraries-arriesgado/
  - name: Upload libraries as artifact
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
  name: libraries-arriesgado
  path: libraries-arriesgado.tar.gz
@@ -103,11 +103,11 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - name: Download libraries
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
  with:
  name: libraries
  - name: Unpack libraries
@@ -131,10 +131,10 @@ jobs:
  - name: Clean workspace
  run: |
  rm -rf libraries library-build testpackage
- rm -f libraries.tar.zst dtestpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt
+ rm -f libraries.tar.zst testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt
  rm -f *.xml
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - name: Make clean
@@ -152,13 +152,13 @@ jobs:
  retry_on: error
  command: ls vlasiator
  - name: Upload testpackage binary
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
  name: vlasiator-testpackage
  path: vlasiator
  if-no-files-found: error
  #- name: Upload build log
- # uses: actions/upload-artifact@v3
+ # uses: actions/upload-artifact@v4
  # with:
  # name: Testpackage build log
  # path: build.log
@@ -169,7 +169,7 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - name: Clean workspace
@@ -178,7 +178,7 @@ jobs:
  rm -rf stdout.txt stderr.txt metrics.txt
  rm -rf *.xml
  - name: Download libraries
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
  with:
  name: libraries-arriesgado
  - name: Unpack libraries
@@ -190,7 +190,7 @@ jobs:
  #srun --interactive -p arriesgado-jammy -J vlasiator_build -n 1 -c 4 --pty -t 01:00:00 bash -lc 'module load boost papi openmpi; export VLASIATOR_ARCH=arriesgado; make -j4'
  ssh synth-hca 'source /etc/profile.d/lmod.sh; export PATH=$PATH:$HOME/bin; module load llvm/cross/EPI-0.7-development; export VLASIATOR_ARCH=arriesgado; cd '$GITHUB_WORKSPACE'; make -j 12'
  - name: Upload riscv binary
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
  name: vlasiator-riscv
  path: vlasiator
@@ -202,7 +202,7 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - name: Setup MPI
@@ -221,7 +221,7 @@ jobs:
  brew install boost
  ln -s /opt/homebrew/Cellar/boost/* /opt/homebrew/Cellar/boost/latest
  - name: Download libraries
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
  with:
  name: libraries-appleM1
  - name: Unpack libraries
@@ -241,10 +241,10 @@ jobs:
  - name: Clean workspace
  run: |
  rm -rf libraries library-build testpackage
- rm -f libraries.tar.zstd dtestpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt
+ rm -f libraries.tar.zstd testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt
  rm -f *.xml
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - uses: ursg/gcc-problem-matcher@master
@@ -255,7 +255,7 @@ jobs:
  export VLASIATOR_ARCH=carrington_gcc_openmpi
  srun -M carrington --job-name tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 vlsvextract vlsvdiff'
  - name: Upload tools binaries
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
  name: vlasiator-tools
  path: |
@@ -271,15 +271,15 @@ jobs:
 
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  - name: Download testpackage binary
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
  with:
  name: vlasiator-testpackage
  - name: Download tools
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
  with:
  name: vlasiator-tools
  - name: Run testpackage
@@ -292,18 +292,31 @@ jobs:
  #srun -M carrington -t 01:30:00 --job-name CI_testpackage --interactive --nodes=1 -c 4 -n 16 --mem-per-cpu=5G -p short ./small_test_carrington_github_ci.sh
  VLASIATOR_CIRUNNER_JOBSPEC_TP=$(sbatch --parsable -o testpackage_run_output.txt ./small_test_carrington_github_ci.sh)
  VLASIATOR_CIRUNNER_JOBSPEC_TP=(${VLASIATOR_CIRUNNER_JOBSPEC_TP//\;/ })
- trap "scancel -M ${VLASIATOR_CIRUNNER_JOBSPEC_TP[1]} ${VLASIATOR_CIRUNNER_JOBSPEC_TP[0]}" SIGINT
- srun -M carrington --dependency afterany:${VLASIATOR_CIRUNNER_JOBSPEC_TP[0]} echo "Job finished, checking output."
+ trap "scancel -M carrington ${VLASIATOR_CIRUNNER_JOBSPEC_TP[0]}" SIGINT
+ trap "scancel -M carrington ${VLASIATOR_CIRUNNER_JOBSPEC_TP[0]}" SIGTERM
+ RUN_STRING=$( cat << MORO
+ echo "Job finished, checking output."
  cat testpackage_run_output.txt
  cat $GITHUB_STEP_SUMMARY > $GITHUB_WORKSPACE/testpackage_check_description.txt
  cd $GITHUB_WORKSPACE
- tar -czvf testpackage-output.tar.gz testpackage_check_description.txt testpackage_output_variables.txt
+ ls -halB testpackage_check_description.txt
+ tar -czf testpackage-output.tar.gz testpackage_check_description.txt testpackage_output_variables.txt
+ MORO
+ )
+ srun -M carrington --dependency afterany:${VLASIATOR_CIRUNNER_JOBSPEC_TP[0]} bash -c "$RUN_STRING"
  if [ -f $GITHUB_WORKSPACE/testpackage_failed ]; then
  # Fail this step if any test failed.
  exit 1
  fi
+ - name: Make sure the output tarball is visible in lustre
+ uses: nick-fields/retry@v3
+ with:
+ timeout_seconds: 15
+ max_attempts: 3
+ retry_on: error
+ command: ls $GITHUB_WORKSPACE/testpackage-output.tar.gz
  - name: Upload testpackage output
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  if: always()
  with:
  name: testpackage-output
@@ -317,11 +330,11 @@ jobs:
  needs: [build_libraries]
  steps:
  - name: Checkout source
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
  with:
  submodules: true
  # - name: Download libraries
- # uses: actions/download-artifact@v3
+ # uses: actions/download-artifact@v4
  # with:
  # name: libraries
  # - name: Unpack libraries
@@ -333,9 +346,9 @@ jobs:
  srun -M carrington --job-name iST_compile --interactive --nodes=1 -n 1 -c 3 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 3 main differentialFlux sigmaProfiles'
  # VLASIATOR_ARCH=carrington_gcc_openmpi make -j 3 main differentialFlux sigmaProfiles
  - name: Upload ionosphereTest binaries
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
  with:
- name: vlasiator-tools
+ name: ionosphereTests
  path: |
  mini-apps/ionosphereSolverTests/main
  mini-apps/ionosphereSolverTests/differentialFlux

MAKE/Makefile.MN5_gpp

@@ -0,0 +1,92 @@
+# -*- mode: makefile -*-
+# GCC
+CMP = mpicxx
+LNK = mpicxx
+
+# Intel
+#CMP = mpiicpx
+#LNK = mpiicpx
+
+# Starting Makefile for the general partition of the MareNostrum5 supercomputer located at the BSC.
+
+#======== Vectorization ==========
+# Set vector backend type for vlasov solvers, sets precision and length. 
+# NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
+# Options: 
+# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
+# AVX512: VEC8D_AGNER, VEC16F_AGNER
+# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
+
+ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
+ # Single-precision 
+ VECTORCLASS = VEC16F_AGNER
+else
+ # Double-precision
+ VECTORCLASS = VEC8D_AGNER
+endif
+
+# GNU flags:
+CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx2
+testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mavx
+
+MATHFLAGS = -ffast-math -fno-finite-math-only
+testpackage: MATHFLAGS = -fno-unsafe-math-optimizations
+
+# GCC
+LDFLAGS = -fopenmp -Wl,--as-needed
+LIB_MPI = -lgomp
+
+# Intel
+#LDFLAGS = -qopenmp -liomp -Wl,--as-needed
+
+#======== Allocator =========
+# Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
+# Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
+CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+
+#======= Compiler and compilation flags =========
+# NOTES on compiler flags:
+# CXXFLAGS is for compiler flags, they are always used
+# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
+# LDFLAGS flags for linker
+
+#-DNO_WRITE_AT_ALL: Define to disable write at all to 
+# avoid memleak (much slower IO)
+#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition 
+# errors that come up when using 
+# mpi.h in c++ on Cray
+
+CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK
+testpackage: CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK
+
+# BOOST_VERSION = current trilinos version
+# ZOLTAN_VERSION = current trilinos verson
+
+#======== Libraries ===========
+
+# Adjust to where your own libraries/dependencies have been built.
+LIBRARY_PREFIX = /gpfs/projects/bsc33/bsc033626/LIBS
+
+# Compiled libraries
+INC_BOOST = -I/apps/GPP/BSCTOOLS/deps/boost/1.84.0/include
+LIB_BOOST = -L/apps/GPP/BSCTOOLS/deps/boost/1.84.0/lib -lboost_program_options
+
+INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/VLASIATOR.GCC.DEV/include
+LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/VLASIATOR.GCC.DEV/lib -lzoltan
+
+INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc/VLASIATOR.GCC.DEV-5.3.0/include
+LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/VLASIATOR.GCC.DEV-5.3.0/lib -ljemalloc
+
+INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv/VLASIATOR.GCC.DEV/include
+LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv/VLASIATOR.GCC.DEV/lib -lvlsv
+
+INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/VLASIATOR.GCC.DEV/include
+LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/VLASIATOR.GCC.DEV/lib -lphiprof
+
+# Header libraries
+
+INC_EIGEN = -isystem ./submodules/eigen/
+INC_DCCRG = -I./submodules/dccrg
+INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/
+INC_FSGRID = -I./submodules/fsgrid

MAKE/Makefile.yann

@@ -1,21 +1,21 @@
 # For Yann's laptop
 
 #======== Vectorization ==========
-#Set vector backend type for vlasov solvers, sets precision and length. 
+#Set vector backend type for vlasov solvers, sets precision and length.
 #NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
-#Options: 
+#Options:
 # AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
 # AVX512: VEC8D_AGNER, VEC16F_AGNER
 # Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
 
 ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
-#Single-precision 
+#Single-precision
  INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/
- VECTORCLASS = VEC4F_FALLBACK
+ VECTORCLASS = VEC8F_AGNER
 else
 #Double-precision
  INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/
- VECTORCLASS = VEC4D_FALLBACK
+ VECTORCLASS = VEC4D_AGNER
 endif
 
 
@@ -29,14 +29,25 @@ PAPI_FLAG =
 
 FLAGS =
 #CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx
-CXXFLAGS = -g3 -ggdb -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx
+CXXFLAGS += -g3 -ggdb -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mfma -mavx2 -Wno-unknown-pragmas -Wno-sign-compare
 testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mno-avx -mno-fma -fno-unsafe-math-optimizations
 
 MATHFLAGS = -ffast-math -fno-finite-math-only
 testpackage: MATHFLAGS = -fno-unsafe-math-optimizations
 
 LDFLAGS =
-LIB_MPI = -lgomp
+LIB_MPI = -lgomp -lpapi
+
+#======== PAPI ==========
+#Add PAPI_MEM define to use papi to report memory consumption?
+CXXFLAGS += -DPAPI_MEM
+testpackage: CXXFLAGS += -DPAPI_MEM
+
+#======== Allocator =========
+#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
+#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
+CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 
 
 #======== Libraries ===========

common.h

@@ -1,4 +1,4 @@
-/*
+/* 
  * This file is part of Vlasiator.
  * Copyright 2010-2016 Finnish Meteorological Institute
  *