datatran

#%Module1.0
# The first line of this file tells Modules that this is a module file.
# DO NOT ALTER IT!
#
# METADATA AND DOCUMENTATION SECTION
#
# This function is part of the Modules help system.  You can modify
# the second line if needed, but most products should
# leave this alone.
#
proc ModulesHelp { } {
    global product version
    puts stderr "This module adds $product/$version to your environment."
}
#
# These variables are used below.  The product variable should be set to
# the name of the product and never changed.  The version variable will
# be set at install time, so it should be left alone.  The conflict line
# prevents multiple versions from being loaded simultaneously.  Do not
# change it.
#
set product desimodules
set version datatran
conflict $product

# NOTE: As of 2024-07-19, KPNO sets $DESI_PRODUCT_ROOT and
# "module load desiconda" before loading desimodules,
# in which case this desiconda_version isn't used at all.
# The current structure is a leftover from supporting
# cori+perlmutter, and it might be useful again in the future,
# even if the if blocks are overkill right now.
set desiconda_version 20240425-2.2.0
if { [info exists env(NERSC_HOST)] } {
    if { $env(NERSC_HOST) == "perlmutter" } {
        set desiconda_version 20240425-2.2.0
    }
    if { $env(NERSC_HOST) == "datatran" } {
        set desiconda_version 20220928-2.1.0
    }
}
#
# The line below is another part of the Modules help system.  You can
# modify the part in quotes if you really need to, but most products should
# leave this alone.
#
module-whatis "Sets up $product/$version in your environment."
#
# ENVIRONMENT SECTION
#
# The PRODUCT_ROOT and PRODUCT_DIR variables are used to set other
# environment variables but are not exported to the actual environment.
# If you are not working at NERSC, but still want to use Modules, you
# will need to set the DESI_PRODUCT_ROOT environment variable
#
if {[info exists env(DESI_PRODUCT_ROOT)]} {
    set code_root $env(DESI_PRODUCT_ROOT)/code
} else {
    set code_root /global/common/software/desi/$env(NERSC_HOST)/desiconda/$desiconda_version/code
}
set PRODUCT_DIR /global/common/software/desi/$env(NERSC_HOST)/desiconda/startup/modulefiles/$product
#
# This line creates an environment variable pointing to the install
# directory of your product.
#
setenv [string toupper $product] $PRODUCT_DIR
setenv [string toupper $product]_VERSION $version
#
# desiconda contains the external products that we need for most DESI work.
# The value of $desiconda_version, defined above, may change depending
# on system, e.g. cori v. datatran.
#
if { [info exists env(NERSC_HOST)] } {
    set desiconda_modules_root [string map {/code /modulefiles} $code_root]
    module use $desiconda_modules_root
    module load desiconda/$desiconda_version
    prereq desiconda
}
#
# DESI-specific modules
#
module load desiutil/3.4.2
prereq desiutil

module load desitree/0.7.0
prereq desitree

# Main versions of most modules:
# module load specter/main
# module load gpu_specter/main
# module load desimodel/main
# module load specex/main
# module load desitarget/main
# module load specsim/main
# module load desispec/main
# module load desisim/main
# module load fiberassign/main
# module load desisurvey/main
# module load surveysim/main
# module load redrock/main
# module load redrock-templates/main
# module load prospect/main
# module load desimeter/main
# module load simqso/main
# module load dust/v0_1
# module load speclite/main
# module load QuasarNP/0.2.0
# module load specprod-db/main

#
# Environment variables that point to checkouts of software products, or
# similar calibration products.
#
if { [info exists env(NERSC_HOST) ] } {
    setenv DESI_ROOT_READONLY /dvs_ro/cfs/cdirs/desi
}
setenv DESI_SPECTRO_CALIB $env(DESI_ROOT)/spectro/desi_spectro_calib/trunk
setenv DESI_SPECTRO_DARK $env(DESI_ROOT)/spectro/desi_spectro_dark/latest
setenv DESI_SURVEYOPS $env(DESI_ROOT)/survey/ops/surveyops/trunk
setenv FIBER_ASSIGN_DIR $env(DESI_ROOT)/target/fiberassign/tiles/trunk
setenv QN_MODEL_FILE $env(DESI_ROOT)/target/catalogs/lya/qn_models/qn_train_coadd_indtrain_0_0_boss10.h5
setenv SQ_MODEL_FILE $env(DESI_ROOT)/target/catalogs/lya/sq_models/BOSS_train_64plates_model.json
set dust_version v0_1
setenv DUST_DIR $env(DESI_ROOT)/external/dust/$dust_version
setenv DUST_VERSION $dust_version
#
# NERSC performance settings.
# See e.g., https://docs.nersc.gov/development/languages/python/using-python-perlmutter/#mpi-issues
#
# Although these environment variables are somewhat NERSC-specific, they should
# be harmless if set elsewhere.
#
# May solve some OpenMP instabilities at NERSC, however it is no longer known what those actual instabilities were.
# See discussion in https://github.com/desihub/desimodules/pull/52.
setenv KMP_INIT_AT_FORK FALSE
# As of 2024, this is no longer listed as an ongoing issue in https://docs.nersc.gov/current/#ongoing-issues.
# See discussion in https://github.com/desihub/desimodules/pull/52.
setenv MPI4PY_RC_RECV_MPROBE "False"
# https://docs.nersc.gov/development/languages/python/using-python-perlmutter/#issues-with-fork-in-mpi-processes
setenv CXI_FORK_SAFE 1
setenv CXI_FORK_SAFE_HP 1
# Consolidated from desispec, redrock & fastspecfit module files.
setenv KMP_AFFINITY disabled
# Consolidated from desispec, redrock & fastspecfit module files.
setenv MPICH_GNI_FORK_MODE FULLCOPY
# Consolidated from redrock & fastspecfit module files.
setenv OMP_NUM_THREADS 1
# Moved from redrock module file.
setenv HDF5_USE_FILE_LOCKING FALSE
# Moved from fastspecfit module file.
setenv MKL_NUM_THREADS 1
#
# Settings that can *only* be set at NERSC.
#
if { [info exists env(NERSC_HOST)] } {
    if { $env(NERSC_HOST) == "perlmutter" } {
        # Use a specific version of cudatoolkit.
        module swap cudatoolkit/12.2
        # https://docs.nersc.gov/development/languages/python/using-python-perlmutter/#conda-cray-mpich-libfabric-dependencies-missing-evp-symbols
        module load evp-patch
    }
}