Modification to allow for both standalone and nextsim

NMOa0.08/expt_01.1/blkdat.input

@@ -127,7 +127,7 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    6.0	  'thkriv' = nominal thickness of river inflow (m)
   10.0    'thkcdw' = thickness for near-surface currents in ice-ocean stress (m)
   20.0	  'thkfrz' = maximum thickness of near-surface freezing zone (m)
-   0	  'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf)
+   0	    'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf,3=nextsim)
    0.0    'tfrz_0' = ENLN: ice melting point (degC) at S=0psu
   -0.054  'tfrz_s' = ENLN: gradient of ice melting point (degC/psu)
    0.25   'frzifq' = e-folding time scale back to tfrz (days or -ve time steps)

TP0a1.00/expt_01.0/blkdat.input

@@ -105,7 +105,7 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    6.0	  'thkriv' = nominal thickness of river inflow (m)
    3.0    'thkcdw' = thickness for near-surface currents in ice-ocean stress (m)
   20.0	  'thkfrz' = maximum thickness of near-surface freezing zone (m)
-   2	  'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf)
+   2	    'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf,3=nextsim)
    0.0    'tfrz_0' = ENLN: ice melting point (degC) at S=0psu
   -0.054  'tfrz_s' = ENLN: gradient of ice melting point (degC/psu)
    0.25   'frzifq' = e-folding time scale back to tfrz (days or -ve time steps)

TP0a1.00/expt_01.1/blkdat.input

@@ -105,7 +105,7 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    6.0	  'thkriv' = nominal thickness of river inflow (m)
    3.0    'thkcdw' = thickness for near-surface currents in ice-ocean stress (m)
   20.0	  'thkfrz' = maximum thickness of near-surface freezing zone (m)
-   0	  'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf)
+   0	  'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf,3=nextsim)
    0.0    'tfrz_0' = ENLN: ice melting point (degC) at S=0psu
   -0.054  'tfrz_s' = ENLN: gradient of ice melting point (degC/psu)
    0.25   'frzifq' = e-folding time scale back to tfrz (days or -ve time steps)
@@ -191,7 +191,6 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    1	  'relax ' = activate lateral boundary nudging    (0=F,1=T)
    1	  'trcrlx' = activate lat. bound. tracer nudging  (0=F,1=T)
    1	  'priver' = rivers as a precipitation bogas      (0=F,1=T)
-   0      'triver' = total river including Greenland (0=F,1=T)
    0     'epmass' = treat evap-precip as a mass exchange (0=F,1=T)
    0     'nmrsti' = 'restart'  Restart filename. Dot and date appended. Empty: use old hycom name.
    0     'nmrsto' = 'restart'  Restart filename. Dot and date appended. Empty: use old hycom name.

TP0a1.00/expt_01.1/pbsscript_chpc.sh

@@ -0,0 +1,77 @@
+#!/bin/bash -l
+
+#PBS -P ERTH0904
+#PBS -N AGUa010
+
+#PBS -l select=1:ncpus=4:mpiprocs=4
+
+## System message output file
+#PBS -o log/HYCOM-CICE.out
+
+## System error message file
+#PBS -e log/HYCOM-CICE.err
+
+## How long job takes, wallclock time hh:mm:ss
+#PBS -l walltime=0:30:00
+#PBS -q test
+
+##PBS -m abe
+##PBS -M 
+
+module load chpc/python/3.7.0
+module load chpc/parallel_studio_xe/64/16.0.1/2016.1.150
+module load chpc/compmech/fftw/3.3.10_intel2020u1
+module load chpc/earth/netcdf/4.7.4/intel2020u1
+module load chpc/compmech/netlib-lapack/3.10.1
+module load chpc/earth/udunits2/udunits-2.2.28
+
+export NMPI=$PBS_NP
+
+# Enter directory from where the job was submitted
+cd $PBS_O_WORKDIR       ||  { echo "Could not go to dir $PBS_O_WORKDIR  "; exit 1; }
+
+# ------------------- Fetch Environment ------------------------------
+# -------- these are needed in preprocess scripts---------------------
+echo "PBS_JOBID    = $PBS_JOBID     "
+echo "PBS_O_WORKDIR= $PBS_O_WORKDIR "
+[ -z "$NOMP" ] && NOMP=0
+
+# Enter directory from where the job was submitted
+cd $PBS_O_WORKDIR       ||  { echo "Could not go to dir $PBS_O_WORKDIR  "; exit 1; }
+
+
+# Initialize environment (sets Scratch dir ($S), Data dir $D ++ )
+source ../REGION.src  || { echo "Could not source ../REGION.src "; exit 1; }
+source ./EXPT.src  || { echo "Could not source EXPT.src"; exit 1; }
+echo "NMPI =$NMPI (Number of MPI tasks needed for running job) "
+
+  START="2020-09-15T12:00:00"
+  END="2020-09-19T12:00:00"
+  INITFLG="--init"
+#  INITFLG=""
+#  ./bin/atmo_synoptic.sh era5+lw $START $END 
+#  ./bin/expt_preprocess.sh $START $END $INITFLG
+
+  echo "Start time in srjob.sh: $START"
+  echo "End   time in srjob.sh: $END"
+
+  # Generate atmospheric forcing :
+#  atmo_synoptic.sh era5+lw $START $END 
+
+  # Transfer data files to scratch - must be in "expt_XXX" dir for this script
+  expt_preprocess.sh $START $END $INITFLG        ||  { echo "Preprocess had fatal errors "; exit 1; }
+
+  # Enter Scratch/run dir and Run model
+  cd $S  ||  { echo "Could not go to dir $S  "; exit 1; }
+  mpirun -np $NMPI ./hycom_alone  
+
+  # Cleanup and move data files to data directory - must be in "expt_XXX" dir for this script
+  cd $P     ||  { echo "Could not go to dir $P  "; exit 1; }
+  expt_postprocess.sh 
+
+done
+
+exit $?
+
+
+

TP2a0.10/expt_01.0/blkdat.input

@@ -127,7 +127,7 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    6.0	  'thkriv' = nominal thickness of river inflow (m)
    3.0    'thkcdw' = thickness for near-surface currents in ice-ocean stress (m)
   20.0	  'thkfrz' = maximum thickness of near-surface freezing zone (m)
-   2	  'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf)
+   2	    'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf,3=nextsim)
    0.0    'tfrz_0' = ENLN: ice melting point (degC) at S=0psu
   -0.054  'tfrz_s' = ENLN: gradient of ice melting point (degC/psu)
    0.25   'frzifq' = e-folding time scale back to tfrz (days or -ve time steps)
@@ -213,7 +213,6 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    1	  'relax ' = activate lateral boundary nudging    (0=F,1=T)
    0	  'trcrlx' = activate lat. bound. tracer nudging  (0=F,1=T)
    1	  'priver' = rivers as a precipitation bogas      (0=F,1=T)
-   0      'triver' = total river including Greenland      (0=F,1=T)
    0      'epmass' = treat evap-precip as a mass exchange (0=F,1=T)
    0      'nmrsti' = 'restart'  Restart filename. Dot and date appended. Empty: use old hycom name.
    0      'nmrsto' = 'restart'  Restart filename. Dot and date appended. Empty: use old hycom name.

TP5a0.06/expt_01.0/blkdat.input

@@ -127,7 +127,7 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    6.0    'thkriv' = nominal thickness of river inflow (m)
   10.0    'thkcdw' = thickness for near-surface currents in ice-ocean stress (m)
   20.0    'thkfrz' = maximum thickness of near-surface freezing zone (m)
-   2      'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf)
+   2      'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf,3=nextsim)
    0.0    'tfrz_0' = ENLN: ice melting point (degC) at S=0psu
   -0.054  'tfrz_s' = ENLN: gradient of ice melting point (degC/psu)
    0.25   'frzifq' = e-folding time scale back to tfrz (days or -ve time steps)
@@ -213,7 +213,6 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    1      'relax ' = activate lateral boundary nudging    (0=F,1=T)
    0      'trcrlx' = activate lat. bound. tracer nudging  (0=F,1=T)
    0      'priver' = rivers as a precipitation bogas      (0=F,1=T)
-   0      'triver' = total river including Greenland      (0=F,1=T)
    0     'epmass' = treat evap-precip as a mass exchange (0=F,1=T)
    1   'nmrsti' = 'TP5restart'  Restart filename. Dot and date appended. Empty: use old hycom name.
    1   'nmrsto' = 'TP5restart'  Restart filename. Dot and date appended. Empty: use old hycom name.

TP5a0.06/expt_01.1/blkdat.input

@@ -127,7 +127,7 @@ S-Z(15-11): dp00/f/x/i=3m/1.125/12m/1m; ds=1m/1.125/4m; src_2.2.12;
    6.0	  'thkriv' = nominal thickness of river inflow (m)
    3.0    'thkcdw' = thickness for near-surface currents in ice-ocean stress (m)
   20.0	  'thkfrz' = maximum thickness of near-surface freezing zone (m)
-   2	  'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf)
+   2	    'iceflg' = sea ice model flag (0=none,1=energy loan,2=coupled/esmf,3=nextsim)
    0.0    'tfrz_0' = ENLN: ice melting point (degC) at S=0psu
   -0.054  'tfrz_s' = ENLN: gradient of ice melting point (degC/psu)
    0.25   'frzifq' = e-folding time scale back to tfrz (days or -ve time steps)

bin/atmo_synoptic.sh

@@ -48,6 +48,12 @@ if [ ${forcing:0:4} == "erai" ] ; then
    if [[ -n $ERAI_PATH ]] ; then
       ROOTPATH=$ERAI_PATH
    fi
+elif [ ${forcing:0:9} == "era5_chpc" ] ; then
+   xmlfile=$INPUTDIR/era5_chpc.xml
+   # if ERA5_PATH is set, it will override rootPath in xml file
+   if [[ -n $ERA5_PATH ]] ; then
+      ROOTPATH=$ERA5_PATH
+   fi
 elif [ ${forcing:0:4} == "era5" ] ; then
    xmlfile=$INPUTDIR/era5.xml
    # if ERA5_PATH is set, it will override rootPath in xml file

bin/common_functions.sh

@@ -293,6 +293,7 @@ tellerror () {
   echo "[FATAL  ] $1" 
   let numerr=$numerr+1; 
   #echo "[FATAL  ] $1" >> $logfile
+  FATALERROR=$FATALERROR:$1
 }
 
 tellwarn () {

bin/compile_model.sh

@@ -142,6 +142,9 @@ elif [ "${hostnamed:0:5}" == "betzy" ] ; then
 elif [ "${hostnamed:0:4}" == "fram" ] ; then # fram
    SITE="fram"
    MACROID=$ARCH.$SITE.$compiler
+elif [ "${hostnamed:0:4}" == "cm.c" ] ; then # fram
+   SITE="chpc"
+   MACROID=$ARCH.$SITE.$compiler
 # Generic case. SITE is empty
 elif [[ "${ARCH}" == "Linux" ]] ; then
    SITE=""
@@ -195,11 +198,14 @@ elif [ "$SITE" == "surfsara" ] ; then
    export ESMF_MOD_DIR=${ESMF_DIR}mod/
    export ESMF_LIB_DIR=${ESMF_DIR}lib/
 
+elif [[ "${unames:0:5}" == "Linux" ]] && [[ "$SITE" == "chpc" ]] ; then
+   export ESMF_DIR=#${EBROOTESMF}/
+   export ESMF_MOD_DIR=#${ESMF_DIR}mod/
+   export ESMF_LIB_DIR=#${ESMF_DIR}lib/
 elif [[ "${unames:0:5}" == "Linux" ]] && [[ "$SITE" == "fram" ]] ; then
    export ESMF_DIR=${EBROOTESMF}/
    export ESMF_MOD_DIR=${ESMF_DIR}mod/
    export ESMF_LIB_DIR=${ESMF_DIR}lib/
-
 # If site is not given, try to use a generic setup. Macro names composed of compiler name and mpi lib name (openmpi, mpich, lam, etc etc(
 elif [[ "${unames:0:5}" == "Linux" ]] && [[ "$SITE" == "" ]] ; then
    if [ -z "${ESMF_DIR}" ] ; then
@@ -345,12 +351,25 @@ if [ $ICEFLG -ne 0 ] ; then
         echo "Error when compiling HYCOM, see above "
         exit $res
     fi
-else
-    echo "Now compiling hycom in $targetdir."
+elif [ $ICEFLG -eq 3 ] ; then
+    echo "Now compiling hycom with oasis in $targetdir."
     export ICEFLG=${ICEFLG}
     export ARCH=${MACROID}
     echo $ARCH
 
+    csh Make_hycom.csh ${MACROID} ${ICEFLG}
+    res=$?
+    if [ $res -ne 0 ] ; then
+        echo
+        echo "Error when compiling HYCOM, see above "
+        exit $res
+    fi
+else
+    echo "Now compiling hycom alone in $targetdir."
+    export ICEFLG=${ICEFLG}
+    export ARCH=${MACROID}
+    echo $ARCH
+
     csh Make_hycom.csh ${MACROID} ${ICEFLG}
     res=$?
     if [ $res -ne 0 ] ; then

bin/create_ref_case.sh

@@ -13,8 +13,9 @@ else
    echo "Could not find EXPT.src. This script must be run in expt dir"
    exit 1
 fi
+
 EDIR=$(pwd)/                           # Location of this script
-BASEDIR=$(cd $(dirname $0)/.. && pwd)/ # Location of basedir
+BASEDIR=$(cd .. && pwd)/               # Location of basedir
 source $BASEDIR/REGION.src
 source $EDIR/EXPT.src
 
@@ -113,7 +114,7 @@ res=$?
 [ $res -ne 0 ] && echo "Failure..."
 echo ".."
 
-iceclim=1
+iceclim=0
 # Create a climatology ice cover used by initialization
 cd $EDIR
 echo "Prepare the sea ice cover from climatology:"
@@ -140,14 +141,17 @@ fi
 # Create simple river forcing
 cd $EDIR
 echo "river forcing, if biology active, may take some time"
-if [ $NTRACR -ne 0 ] ; then
-   river_nersc.sh 100 300 $INPUTDIR/rivers_ahype-ehype_clim_rev2.dat $INPUTDIR/biorivers.dat > $EDIR/log/ref_river_nersc.out 2>&1
-   riverfolder=$(echo $X | cut -c1-2)$(echo $X | cut -c4)
-   python $BINDIR/spread_Ob_river_nutrients.py $BASEDIR/force/rivers/${riverfolder}/ > $EDIR/log/spread_river.out 2>&1  # Spreads Ob River nutrients to outer bay
-   python $BINDIR/add_atmdep_to_river.py $BASEDIR/force/rivers/${riverfolder}/  $INPUTDIR/emep_2010_annual_1degree_rv4_17gfecl1p0.nc  > $EDIR/log/add_atmospheric_deposition.out 2>&1
-else
-   river_nersc.sh 100 300 $INPUTDIR/rivers_ahype-ehype_clim_rev2.dat > $EDIR/log/ref_river_nersc.out 2>&1
-fi
+./bin/river_trip.sh  -t 300 -n 150 era5  > $EDIR/log/ref_river_trip.out 2>&1 
+cp ${BASEDIR}/force/rivers/$E/forcing.rivers.a ${BASEDIR}/force/rivers/$E/rivers.a
+cp ${BASEDIR}/force/rivers/$E/forcing.rivers.b ${BASEDIR}/force/rivers/$E/rivers.b
+#if [ $NTRACR -ne 0 ] ; then
+#   river_nersc.sh 100 300 $INPUTDIR/rivers_ahype-ehype_clim_rev2.dat $INPUTDIR/biorivers.dat > $EDIR/log/ref_river_nersc.out 2>&1
+#   riverfolder=$(echo $X | cut -c1-2)$(echo $X | cut -c4)
+#   python $BINDIR/spread_Ob_river_nutrients.py $BASEDIR/force/rivers/${riverfolder}/ > $EDIR/log/spread_river.out 2>&1  # Spreads Ob River nutrients to outer bay
+#   python $BINDIR/add_atmdep_to_river.py $BASEDIR/force/rivers/${riverfolder}/  $INPUTDIR/emep_2010_annual_1degree_rv4_17gfecl1p0.nc  > $EDIR/log/add_atmospheric_deposition.out 2>&1
+#else
+#   river_nersc.sh 100 300 $INPUTDIR/rivers_ahype-ehype_clim_rev2.dat > $EDIR/log/ref_river_nersc.out 2>&1
+#fi
 
 res=$?
 [ $res -eq 0 ] && echo "Success"

bin/expt_preprocess.sh

@@ -240,7 +240,7 @@ echo "tstop  is $tstop"
 echo "--------------------"
 
 # Check that start time is in September when starting from climatology                                                           
-if [ "$initstr" == "--init" ] ;then
+if [ "$initstr" == "--init" && $ICEFLG != 0] ;then
     if [ "$start_month" != "09" ]; then
         tellerror "We recommend starting the model in September when starting from \                                             
 climatology. You can override September initilization by commenting out \                                                        
@@ -289,8 +289,9 @@ ${pget} $BASEDIR/topo/regional.grid.b regional.grid.b || tellerror "no grid file
 ${pget} $BASEDIR/topo/depth_${R}_${T}.a regional.depth.a || tellerror "no topo file depth_${R}_${T}.a" 
 ${pget} $BASEDIR/topo/depth_${R}_${T}.b regional.depth.b || tellerror "no topo file depth_${R}_${T}.b" 
 ${pget} $BASEDIR/topo/kmt_${R}_${T}.nc cice_kmt.nc     || tellerror "no kmt file $BASEDIR/topo/kmt_${R}_${T}.nc "
-${pget} $BASEDIR/topo/cice_grid.nc cice_grid.nc        || tellerror "no cice grid file $BASEDIR/topo/cice_grid.nc "
-
+if [ $ICEFLG != 0 ] ; then
+   ${pget} $BASEDIR/topo/cice_grid.nc cice_grid.nc        || tellerror "no cice grid file $BASEDIR/topo/cice_grid.nc "
+fi
 
 if [ "$SSTRLX" -eq 3 ] ; then
    [ -f  $CLMDIR/seatmp.a ] || tellerror "File $CLMDIR/seatmp.a does not exist"
@@ -714,9 +715,12 @@ compdir=$P/build/${compdir}
 if [ $ICEFLG -eq 2 ] ; then
   echo "Retrieving  hycom_cice from $compdir"
   /bin/cp $compdir/hycom_cice  . || tellerror "Could not get hycom_cice executable at "
-elif [ $ICEFLG -eq 0 ] ; then
+elif [ $ICEFLG -eq 3 ] ; then
   echo "Retrieving  hycom_oasis from $compdir"
   /bin/cp $compdir/hycom_oasis  . || tellerror "Could not get hycom_oasis executable at "
+elif [ $ICEFLG -eq 0 ] ; then
+  echo "Retrieving  hycom_alone from $compdir"
+  /bin/cp $compdir/hycom_alone  . || tellerror "Could not get hycom_alone executable at "
 fi
 
 
@@ -786,7 +790,9 @@ fi
 if [ $numerr -eq 0 ] ; then
    echo "No fatal errors. Ok to start model set up in $S"
 else
-   echo "Some fatal errors occured. See above"
+   echo "Some fatal errors occured. The error message is: "
+   echo $FATALERROR
+   echo "For more details see above."
 fi
 
 # Tell where stuff ended up