Add MPI work manager example based on the basic_nacl_amber example.

basic_nacl_amber_MPI-WM/1.clean.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rm *log *out *.h5
+
+rm -rf seg_logs/ traj_segs/ istates/

basic_nacl_amber_MPI-WM/README.md

@@ -0,0 +1,15 @@
+# Basic WESTPA tutorial: Na+ Cl- association
+This turorial is meant to introduce new users to the WESTPA software for running Weighted Ensemble (WE) simulations.
+
+## Tutorial files
+
+All files necessary for completing the tutorial can be found in this directory.
+
+## Full tutorial 
+
+Find the full tutorial in the LiveComs journal article on best practices for Weighted Ensemble simulations. (Link to be added)
+
+## Authors
+
+* **Anthony Bogetti** - *Primary work* - [atbogetti](https://github.com/atbogetti)
+* **Alex DeGrave** - *Preliminary work* - [ajd98](https://github.com/ajd98)

basic_nacl_amber_MPI-WM/bstates/bstates.txt

@@ -0,0 +1 @@
+0 1 bstate.rst

basic_nacl_amber_MPI-WM/bstates/cpptraj.log

@@ -0,0 +1,8 @@
+# 08/15/19 11:52:14
+# -p ../common_files/nacl.parm7
+# Loaded topologies:
+#   ../common_files/nacl.parm7
+trajin bstate.rst
+distance :1 :2 out dist.out
+go
+quit

basic_nacl_amber_MPI-WM/bstates/dist.out

@@ -0,0 +1,2 @@
+#Frame      Dis_00001
+       1      11.2225

basic_nacl_amber_MPI-WM/common_files/md.in

@@ -0,0 +1,22 @@
+5 ps unrestrained NVT production using Langevin thermostat
+&cntrl
+  ntx       = 5,
+  irest     = 1,
+  ig        = RAND,
+  dt        = 0.002,
+  nstlim    = 1000,
+  nscm      = 500,
+  gamma_ln  = 1.0,
+  ntf       = 2,
+  ntc       = 2,
+  ntb       = 1,
+  ntt       = 3,
+  temp0     = 300.0,
+  cut       = 10.0,
+  ntpr      = 5,
+  ntxo      = 2,
+  ntwr      = 2500,
+  ioutfm    = 1,
+  ntwx      = 20,
+  iwrap     = 0,
+&end

basic_nacl_amber_MPI-WM/env.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Set up environment for dynamics
+module purge
+module load gcc/8.2.0
+module load openmpi/4.0.3
+module load amber/18_AMD_gcc-8.2.0
+
+source $AMBERHOME/amber.sh
+
+# Set WESTPA-related variables
+export WEST_SIM_ROOT="$PWD"
+export SIM_NAME=$(basename $WEST_SIM_ROOT)
+
+# Set runtime commands
+export PMEMD=$(which pmemd)
+export CPPTRAJ=$(which cpptraj)
+

basic_nacl_amber_MPI-WM/init.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Set up simulation environment
+source env.sh
+
+# Clean up from previous/ failed runs
+rm -rf traj_segs seg_logs istates west.h5 
+mkdir   seg_logs traj_segs istates
+
+# Set pointer to bstate and tstate
+BSTATE_ARGS="--bstate-file $WEST_SIM_ROOT/bstates/bstates.txt"
+TSTATE_ARGS="--tstate-file $WEST_SIM_ROOT/tstate.file"
+
+# Run w_init
+w_init \
+  $BSTATE_ARGS \
+  $TSTATE_ARGS \
+  --segs-per-state 1 \
+  --work-manager=threads "$@"

basic_nacl_amber_MPI-WM/run.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#SBATCH --job-name=nacl
+#SBATCH --output=slurm.out
+#SBATCH --nodes=8
+#SBATCH --ntasks-per-node=28
+#SBATCH --cluster=mpi
+#SBATCH --partition=opa
+#SBATCH --time=12:00:00
+
+# Make sure environment is set
+source env.sh
+
+export OMPI_MCA_pml=ob1
+export OMPI_MCA_btl="self,tcp"
+export OMPI_MCA_opal_warn_on_missing_libcuda=0
+
+which mpirun
+echo $SLURM_NTASKS
+echo $WEST_ROOT
+echo $WEST_PYTHON
+echo $WEST_BIN
+
+# Clean up
+rm -f west.log
+
+# Run w_run
+mpirun -n $SLURM_NTASKS \
+          w_run --work-manager=mpi "$@" &> west.log

basic_nacl_amber_MPI-WM/tstate.file

@@ -0,0 +1 @@
+bound 2.6

basic_nacl_amber_MPI-WM/west.cfg

@@ -0,0 +1,91 @@
+# The master WEST configuration file for a simulation.
+# vi: set filetype=yaml :
+---
+west: 
+  system:
+    driver: westpa.core.systems.WESTSystem
+    system_options:
+      # Dimensionality of your progress coordinate
+      pcoord_ndim: 1
+      # Number of data points per iteration
+      pcoord_len: 51
+      # Data type for your progress coordinate 
+      pcoord_dtype: !!python/name:numpy.float32
+      bins:
+        type: RectilinearBinMapper
+        # The edges of the bins 
+        boundaries:         
+          -  [ 0.00, 2.60, 2.80, 3.00, 3.20, 3.40, 3.60, 3.80,
+               4.00, 4.50, 5.00, 5.50, 6.00, 7.00, 8.0, 9.0, 10.0,
+               11.0, 12.0, 13.0, 14.0, 15.0, 'inf']
+      # Number walkers per bin
+      bin_target_counts: 5
+  propagation:
+    max_total_iterations: 100
+    max_run_wallclock:    72:00:00
+    propagator:           executable
+    gen_istates:          false
+  data:
+    west_data_file: west.h5
+    datasets:
+      - name:        pcoord
+        scaleoffset: 4
+      - name:        coord
+        dtype:       float32
+        scaleoffset: 3
+    data_refs:
+      segment:       $WEST_SIM_ROOT/traj_segs/{segment.n_iter:06d}/{segment.seg_id:06d}
+      basis_state:   $WEST_SIM_ROOT/bstates/{basis_state.auxref}
+      initial_state: $WEST_SIM_ROOT/istates/{initial_state.iter_created}/{initial_state.state_id}.rst
+  plugins:
+  executable:
+    environ:
+      PROPAGATION_DEBUG: 1
+    datasets:
+      - name:    coord
+        enabled: false
+    propagator:
+      executable: $WEST_SIM_ROOT/westpa_scripts/runseg.sh
+      stdout:     $WEST_SIM_ROOT/seg_logs/{segment.n_iter:06d}-{segment.seg_id:06d}.log
+      stderr:     stdout
+      stdin:      null
+      cwd:        null
+      environ:
+        SEG_DEBUG: 1
+    get_pcoord:
+      executable: $WEST_SIM_ROOT/westpa_scripts/get_pcoord.sh
+      stdout:     /dev/null 
+      stderr:     stdout
+    gen_istate:
+      executable: $WEST_SIM_ROOT/westpa_scripts/gen_istate.sh
+      stdout:     /dev/null 
+      stderr:     stdout
+    post_iteration:
+      enabled:    true
+      executable: $WEST_SIM_ROOT/westpa_scripts/post_iter.sh
+      stderr:     stdout
+    pre_iteration:
+      enabled:    false
+      executable: $WEST_SIM_ROOT/westpa_scripts/pre_iter.sh
+      stderr:     stdout
+  # Settings for w_ipa, an interactive analysis program that can also automate analysis.
+  analysis:
+     directory: ANALYSIS                # specify the directory all analysis files should exist in.
+     kinetics:                          # general options for both kinetics routines.
+       step_iter: 1 
+       evolution: cumulative
+       extra: [ 'disable-correl' ]
+     analysis_schemes:                  # Analysis schemes.  Required: name (TEST), states, and bins
+       TEST:
+         enabled: True
+         bins:
+           - type: RectilinearBinMapper
+             boundaries: 
+               - [0.0,2.6,10.0,'inf']
+         states:
+           - label: bound
+             coords: 
+               - [0]
+           - label: unbound
+             coords: 
+               - [10.1]  

basic_nacl_amber_MPI-WM/westpa_scripts/cat_trajectory.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+import h5py, numpy, sys
+
+infile = numpy.loadtxt(sys.argv[1], usecols = (0, 1))
+west   = h5py.File('west.h5')
+coords = []
+for iteration, seg_id in infile[1:]:
+    iter_key = "iter_{0:08d}".format(int(iteration))
+    SOD      = west['iterations'][iter_key]['auxdata']['coord'][seg_id,1:,0,:]
+    CLA      = west['iterations'][iter_key]['auxdata']['coord'][seg_id,1:,1,:]
+    coords  += [numpy.column_stack((SOD, CLA))]
+with open(sys.argv[1][:-4] + ".xyz", 'w') as outfile:
+    for i, frame in enumerate(numpy.concatenate(coords)):
+        outfile.write("2\n")
+        outfile.write("{0}\n".format(i))
+        outfile.write("SOD {0:9.5f} {1:9.5f} {2:9.5f}\n".format(
+          float(frame[0]), float(frame[1]), float(frame[2])))
+        outfile.write("CLA {0:9.5f} {1:9.5f} {2:9.5f}\n".format(
+          float(frame[3]), float(frame[4]), float(frame[5])))

basic_nacl_amber_MPI-WM/westpa_scripts/gen_istate.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ -n "$SEG_DEBUG" ] ; then
+    set -x
+    env | sort
+fi
+
+cd $WEST_SIM_ROOT
+
+mkdir -p $(dirname $WEST_ISTATE_DATA_REF)
+ln -s $WEST_BSTATE_DATA_REF $WEST_ISTATE_DATA_REF
+

basic_nacl_amber_MPI-WM/westpa_scripts/get_pcoord.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+if [ -n "$SEG_DEBUG" ] ; then
+  set -x
+  env | sort
+fi
+
+cd $WEST_SIM_ROOT
+
+DIST=$(mktemp)
+
+COMMAND="           parm $WEST_SIM_ROOT/common_files/nacl.parm7 \n"
+COMMAND="${COMMAND} trajin $WEST_STRUCT_DATA_REF \n"
+COMMAND="${COMMAND} distance na-cl :1@Na+ :2@Cl- out $DIST \n"
+COMMAND="${COMMAND} go"
+
+echo -e "${COMMAND}" | $CPPTRAJ
+
+cat $DIST | tail -n +2 | awk '{print $2}' > $WEST_PCOORD_RETURN
+
+rm $DIST
+
+if [ -n "$SEG_DEBUG" ] ; then
+  head -v $WEST_PCOORD_RETURN
+fi

basic_nacl_amber_MPI-WM/westpa_scripts/node.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# node.sh
+
+cd $WEST_SIM_ROOT
+source env.sh
+
+set -x
+$WEST_ROOT/bin/w_run "$@"

basic_nacl_amber_MPI-WM/westpa_scripts/post_iter.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ -n "$SEG_DEBUG" ] ; then
+    set -x
+    env | sort
+fi
+
+cd $WEST_SIM_ROOT || exit 1
+
+ITER=$(printf "%06d" $WEST_CURRENT_ITER)
+tar -cf seg_logs/$ITER.tar seg_logs/$ITER-*.log
+rm  -f  seg_logs/$ITER-*.log

basic_nacl_amber_MPI-WM/westpa_scripts/runseg.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+if [ -n "$SEG_DEBUG" ] ; then
+  set -x
+  env | sort
+fi
+
+cd $WEST_SIM_ROOT
+mkdir -pv $WEST_CURRENT_SEG_DATA_REF
+cd $WEST_CURRENT_SEG_DATA_REF
+
+ln -sv $WEST_SIM_ROOT/common_files/nacl.parm7 .
+
+if [ "$WEST_CURRENT_SEG_INITPOINT_TYPE" = "SEG_INITPOINT_CONTINUES" ]; then
+  sed "s/RAND/$WEST_RAND16/g" $WEST_SIM_ROOT/common_files/md.in > md.in
+  ln -sv $WEST_PARENT_DATA_REF/seg.rst ./parent.rst
+elif [ "$WEST_CURRENT_SEG_INITPOINT_TYPE" = "SEG_INITPOINT_NEWTRAJ" ]; then
+  sed "s/RAND/$WEST_RAND16/g" $WEST_SIM_ROOT/common_files/md.in > md.in
+  ln -sv $WEST_PARENT_DATA_REF ./parent.rst
+fi
+
+$PMEMD -O -i md.in   -p nacl.parm7  -c parent.rst \
+          -r seg.rst -x seg.nc      -o seg.log    -inf seg.nfo
+
+TEMP=$(mktemp)
+COMMAND="         parm nacl.parm7\n"
+COMMAND="$COMMAND trajin $WEST_CURRENT_SEG_DATA_REF/parent.rst\n"
+COMMAND="$COMMAND trajin $WEST_CURRENT_SEG_DATA_REF/seg.nc\n"
+COMMAND="$COMMAND autoimage fixed Na+ \n"
+COMMAND="$COMMAND distance na-cl :1@Na+ :2@Cl- out $TEMP\n"
+COMMAND="$COMMAND go\n"
+
+echo -e $COMMAND | $CPPTRAJ
+cat $TEMP | tail -n +2 | awk '{print $2}' > $WEST_PCOORD_RETURN
+
+if [ ${WEST_COORD_RETURN} ]; then
+  COMMAND="         parm nacl.parm7\n"
+  COMMAND="$COMMAND trajin  $WEST_CURRENT_SEG_DATA_REF/parent.rst\n"
+  COMMAND="$COMMAND trajin  $WEST_CURRENT_SEG_DATA_REF/seg.nc\n"
+  COMMAND="$COMMAND strip :WAT \n"
+  COMMAND="$COMMAND autoimage fixed Na+ \n"
+  COMMAND="$COMMAND trajout $WEST_CURRENT_SEG_DATA_REF/seg.pdb\n"
+  COMMAND="$COMMAND go\n"
+  echo -e $COMMAND | $CPPTRAJ 
+  cat $WEST_CURRENT_SEG_DATA_REF/seg.pdb | grep 'ATOM' \
+    | awk '{print $6, $7, $8}' > $WEST_COORD_RETURN
+fi
+
+# Clean up
+rm -f $TEMP md.in parent.rst seg.nfo seg.pdb nacl.parm7

basic_nacl_amber_MPI-WM/westpa_scripts/tar_segs.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+[ -z "$WEST_SIM_ROOT" ] &&
+    exit 1
+[ ! -d $WEST_SIM_ROOT/traj_segs ] &&
+    exit 1
+
+cd $WEST_SIM_ROOT/traj_segs
+
+ITERS=($(ls | grep '^[0-9][0-9][0-9][0-9][0-9][0-9]$'))
+ITERS=("${ITERS[@]:0:${#ITERS[@]}-1}")
+for ITER in ${ITERS[@]}; do
+    [ ! -f $ITER.tar ] &&
+        tar -cvf $ITER.tar $ITER
+done
+