JURECA Deployment

Instructions for Deploying Software Stack

• Make sure SSH key from JURECA account is added to bbpcode and SSH config is in place:

$ cat ~/.ssh/config

Host bbpcode.epfl.ch
    HostName bbpcode.epfl.ch
    User GASPER_USERNAME

• Install simulation stack using script :

#!/bin/bash

# pass the argument of partition
if [ "$1" == "" ]; then
    echo "Error : pass --cluster or --booster"
    exit 1
fi

module --force purge
module use /usr/local/software/jureca/OtherStages

system=
cnrn_variant=

while [ "$1" != "" ]; do
    case $1 in
        -c | --cluster)         shift
                                system=cluster
                                cnrn_variant=~knl
                                module load Architecture/Haswell
                                module load Stages/2019a
                                module load Intel ParaStationMPI/5.2.2-1-mt HDF5
                                ;;
        -b | --booster )        system=booster
                                cnrn_variant=+knl
                                module load Architecture/KNL
                                module load Stages/2019a
                                module load Intel ParaStationMPI/5.2.2-1-mt HDF5
                                ;;
        * )                     echo "Error : --cluster or --booster"
                                exit 1
    esac
    shift
done

set -e

# Deployment directory
date=$(date '+%d-%m-%Y')

DEPLOYMENT_HOME=`pwd`/HBP/jureca-$system/$date
mkdir -p $DEPLOYMENT_HOME/sources

# Clone spack repository and setup environment
cd $DEPLOYMENT_HOME/sources
git clone https://github.com/BlueBrain/spack.git
export SPACK_ROOT=`pwd`/spack
export PATH=$SPACK_ROOT/bin:$PATH
source $SPACK_ROOT/share/spack/setup-env.sh

# Copy configurations
mkdir -p $SPACK_ROOT/etc/spack/defaults/linux/
cp $SPACK_ROOT/sysconfig/jureca-$system/* $SPACK_ROOT/etc/spack/defaults/linux/

# Directory for deployment
export SPACK_INSTALL_PREFIX=$DEPLOYMENT_HOME

# Install packages
spack spec -I py-bluepy%gcc ^[email protected] ^[email protected]
spack install --dirty --keep-stage -v py-bluepy%gcc ^[email protected] ^[email protected]
spack spec -I -l py-bluepyopt%gcc^neuron~binary~mpi ^[email protected] ^[email protected]
spack install --dirty --keep-stage -v py-bluepyopt%gcc^neuron~binary~mpi ^[email protected] ^[email protected]
spack spec -I neurodamus-hippocampus%intel+coreneuron^coreneuron${cnrn_variant} ^[email protected] ^[email protected]
spack install --dirty --keep-stage -v neurodamus-hippocampus+coreneuron%intel^coreneuron${cnrn_variant} ^[email protected] ^[email protected]

module load Python/3.6.8
spack spec -I neuron~mpi ^[email protected]
spack install --dirty --keep-stage -v neuron~mpi ^[email protected]

# Re-generate modules
spack module tcl refresh --delete-tree -y

# Create symbolic link
mkdir -p $DEPLOYMENT_HOME/../install && cd $DEPLOYMENT_HOME/../install
rm -f latest
ln -s $date/install/modules/tcl/linux-centos7-x86_64 latest

and invoke it as:

$ bash -x hbp-cluster.sh --booster  # for booster partition

# or

$ bash -x hbp-cluster.sh --cluster  # for cluster partition

• Notify user for new deployed modules

 export MODULEPATH=$MODULEPATH:/p/project/cvsk25/software-deployment/HBP/jureca-cluster/install/latest
# or
 export MODULEPATH=$MODULEPATH:/p/project/cvsk25/software-deployment/HBP/jureca-booster/install/latest

Instructions for Using Software Stack

• Sample job script for cpu/cluster partition

#!/bin/bash
#SBATCH --nodes=100
#SBATCH --ntasks-per-node=24
#SBATCH --job-name=test-hippo-cluster
#SBATCH --time=6:00:00
#SBATCH --partition=batch
#SBATCH --mail-user=email
#SBATCH --mail-type=ALL
#SBATCH --account=vsk25


module --force purge all
module use /usr/local/software/jureca/OtherStages
module load Architecture/Haswell
module load Stages/2019a
module load Intel ParaStationMPI/5.2.2-1-mt imkl
module load Boost Python/2.7.16 HDF5

export MODULEPATH=/p/project/cvsk25/software-deployment/HBP/jureca-cluster/install/latest:$MODULEPATH

module load neurodamus-hippocampus
module list

export I_MPI_DEBUG=5
srun --cpus-per-task=1 special -NFRAME 1024 $HOC_LIBRARY_PATH/init.hoc -mpi

• Sample job script for knl/booster partition

#!/bin/bash
#SBATCH --nodes=100
#SBATCH --ntasks-per-node=68
#SBATCH --job-name=test-full
#SBATCH --time=12:00:00
#SBATCH --partition=largebooster
#SBATCH --mail-user=email
#SBATCH --mail-type=ALL
#SBATCH --account=vsk25

module --force purge all
module use /usr/local/software/jurecabooster/OtherStages
module load Architecture/KNL
module load Stages/2019a
module load Intel ParaStationMPI/5.2.2-1-mt imkl
module load HDF5 Boost Python/2.7.16

export MODULEPATH=/p/project/cvsk25/software-deployment/HBP/jureca-booster/install/latest:$MODULEPATH

module load neurodamus-hippocampus
module list

export I_MPI_DEBUG=5
srun --cpus-per-task=1 special -NFRAME 1024 $HOC_LIBRARY_PATH/init.hoc -mpi