Merge branch 'main' into vlct-cosmology

.gitignore

@@ -22,6 +22,7 @@ charmrun
 config.log
 errors.org
 warnings.org
+doc/dox-xml
 
 # ignore directories used for building branches
 build-*/Cello/*

CMakeLists.txt

@@ -109,6 +109,8 @@ else()
 endif()
 add_compile_definitions(SMALL_INTS)
 
+add_compile_definitions(CELLO_VERSION="${CMAKE_PROJECT_VERSION}")
+
 # Whether to bypass passing MsgRefine directly to Block constructor,
 # or request it from a separate entry method to bypass a Charm++
 # memory leak. This should only be set to 0 after (and if) the bug is
@@ -342,4 +344,4 @@ endif()
 # extract compile defs from from Cello to populate config
 get_directory_property( CELLO_CPPDEFINES DIRECTORY src/Cello COMPILE_DEFINITIONS )
 # now generate the the config
-configure_file(auto_config.def.in auto_config.def @ONLY)
+configure_file(auto_config.def.in auto_config.def ESCAPE_QUOTES @ONLY)

config/expanse_icc.py

@@ -0,0 +1,75 @@
+import os
+from _common_search_paths import charm_path_search, grackle_path_search
+
+is_arch_valid = 1
+use_gfortran = 0
+smp = 0
+
+flags_arch = '-Wall -O3 -g'
+#flags_arch = '-fprofile-arcs -ftest-coverage'
+#flags_arch = '-Wall -g'
+#flags_arch = '-Wall -g -fsanitize=address -fno-omit-frame-pointer'
+#flags_arch = '-Wall -O3 -pg'
+
+# rdynamic required for backtraces
+#flags_link_charm = '-rdynamic' 
+#flags_link_charm = '-memory paranoid' 
+
+intel_dir = '/cm/shared/apps/spack/cpu/opt/spack/linux-centos8-zen/gcc-8.3.1/intel-19.1.1.217-4d42ptjd6wsnh5bgbzcv6lp44vxpjwut/compilers_and_libraries_2020.1.217/linux/bin/intel64'
+cc  = intel_dir + '/icc'
+f90 = intel_dir + '/ifort'
+
+flags_prec_single = ''
+flags_prec_double = '-r8'
+
+libpath_fortran = '/cm/shared/apps/spack/cpu/opt/spack/linux-centos8-zen/gcc-8.3.1/intel-19.1.1.217-4d42ptjd6wsnh5bgbzcv6lp44vxpjwut/compilers_and_libraries_2020.1.217/linux/compiler/lib/intel64'
+libs_fortran    = ['ifcore', 'ifport']
+
+
+#USE GFORTRAN INSTEAD OF IFORT
+if use_gfortran:
+    f90 = 'gfortran'
+    libpath_fortran = '/cm/shared/apps/spack/cpu/opt/spack/linux-centos8-zen/gcc-8.3.1/gcc-10.2.0-n7su7jf54rc7l2ozegds5xksy6qhrjin/lib64'
+    libs_fortran = ['gfortran']
+    flags_arch_fortran = '-ffixed-line-length-132'
+    flags_prec_double = '-fdefault-real-8 -fdefault-double-8'
+    flags_arch = '-O3 -Wall'
+    flags_fc = ''
+
+#############################
+
+home = os.getenv('HOME')
+
+charm_path = charm_path_search(home)
+
+use_papi=0
+papi_inc = '/usr/local/include'
+papi_lib = '/usr/local/lib'
+
+boost_path = os.getenv('BOOST_HOME')
+boost_inc  = boost_path + '/include'
+boost_lib  = boost_path + '/lib'
+
+hdf5_inc = os.getenv('HDF5HOME') + '/include'
+if hdf5_inc is None:
+	if os.path.exists('/usr/include/hdf5.h'):
+	        hdf5_inc    = '/usr/include'
+	elif os.path.exists('/usr/include/hdf5/serial/hdf5.h'):
+		hdf5_inc    = '/usr/include/hdf5/serial'
+	else:
+		raise Exception('HDF5 include file was not found.  Try setting the HDF5_INC environment variable such that $HDF5_INC/hdf5.h exists.')
+
+hdf5_lib = os.getenv('HDF5HOME') + '/lib'
+if hdf5_lib is None:
+	if os.path.exists('/usr/lib/libhdf5.a'):
+		hdf5_lib    = '/usr/lib'
+	elif os.path.exists('/usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a'):
+		hdf5_lib    = '/usr/lib/x86_64-linux-gnu/hdf5/serial'
+	else:
+		raise Exception('HDF5 lib file was not found.  Try setting the HDF5_LIB environment variable such that $HDF5_LIB/libhdf5.a exists.')
+
+png_path = os.getenv('LIBPNG_HOME')
+if png_path is None:
+	png_path     = '/lib/x86_64-linux-gnu'
+
+grackle_path = os.getenv('GRACKLE_HOME') 

config/hlrn_gcc.cmake

@@ -0,0 +1,37 @@
+# Machine configs are included twice.
+# First to set compilers, paths, default options.
+# Second to set dependent options (e.g., also depending on defaults set in the main CMakeLists.txt)
+if(NOT __processedUserDefaults)
+
+  message(STATUS "Loading machine configuration for HLRN-IV supercomputer.\n"
+    "This configuration has been tested using the following modules (last verified 2022-06-16):\n"
+    "$ module load gcc/9.3.0 openmpi/gcc.9/3.1.5 hdf5/gcc.9/1.12.0 libpng/1.6.37 boost/1.72.0\n"
+    "and with Charm++ configured as follows:\n"
+    "$ cmake -DNETWORK=mpi -DSMP=OFF -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fortran_COMPILER=mpif90 ..\n"
+    "Note: set PNG_ROOT to LIBPNG_PATH using -D or export.\n")
+
+  set(CMAKE_CXX_COMPILER mpicxx CACHE STRING "")
+  set(CMAKE_C_COMPILER mpicc CACHE STRING "")
+  set(CMAKE_Fortran_COMPILER mpif90  CACHE STRING "")
+  set(CMAKE_Fortran_FLAGS "-ffixed-line-length-132" CACHE STRING "Default Fortran flags")
+
+  # add optional flags to C and C++ compilers that provide useful warnings
+  #set(CMAKE_C_FLAGS "-Wall" CACHE STRING "Default C flags")
+  #set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "Default C++ flags")
+
+  # these flag(s) are currently only used when using openmp-simd optimizations
+  # (to specify available/prefered instruction sets).
+  # This particular value tells the compiler to optimize the code for the
+  # instruction set of the machine used to compile the code.
+  set(CONFIG_ARCH_FLAGS "-march=skylake-avx512")
+
+  # Mark done
+  set(__processedUserDefaults ON)
+
+else()
+
+  if (USE_DOUBLE_PREC)
+    string(APPEND CMAKE_Fortran_FLAGS " -fdefault-real-8 -fdefault-double-8")
+  endif()
+
+endif()

config/linux_gcc.cmake

@@ -9,27 +9,20 @@ if(NOT __processedUserDefaults)
   set(CMAKE_CXX_COMPILER g++ CACHE STRING "")
   set(CMAKE_C_COMPILER gcc CACHE STRING "")
   set(CMAKE_Fortran_COMPILER gfortran CACHE STRING "")
-  # Note (12/2021): passing -march=native to gfortran seems to slow down the
-  # PPM solver
   set(CMAKE_Fortran_FLAGS "-ffixed-line-length-132" CACHE STRING "Default Fortran flags")
 
-  # add optional flags to C and C++ compilers that provide useful warnings
-  set(CMAKE_C_FLAGS "-Wall" CACHE STRING "Default C flags")
-  set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "Default C++ flags")
+  # Set some architecture-specific optimization flags
+  set(__ARCH_C_OPT_FLAGS "-O3 -DNDEBUG -funroll-loops")
 
-
-  # these flag(s) are currently only used when using openmp-simd optimizations
-  # (to specify available/prefered instruction sets).
-  # This particular value tells the compiler to optimize the code for the
-  # instruction set of the machine used to compile the code.
-  set(CONFIG_ARCH_FLAGS "-march=native")
-
-  # maybe we should add the following flags at a higher level...
-  string(APPEND CMAKE_C_FLAGS " -funroll-loops")
-  string(APPEND CMAKE_CXX_FLAGS " -funroll-loops")
+  set(CMAKE_C_FLAGS_RELEASE "${__ARCH_C_OPT_FLAGS}")
+  set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g ${__ARCH_C_OPT_FLAGS}")
+  set(CMAKE_CXX_FLAGS_RELEASE "${__ARCH_C_OPT_FLAGS}")
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g ${__ARCH_C_OPT_FLAGS}")
 
   # Setting package paths (e.g., Grackle)
 
+  set(Grackle_ROOT "/home/regan/data/SourceCodes/Grackle/GRACKLE_INSTALL" CACHE STRING "my grackle build")
+
   # Mark done
   set(__processedUserDefaults ON)
 

doc/source/conf.py

@@ -11,7 +11,7 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os, subprocess
+import json, sys, os, subprocess
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -276,16 +276,53 @@
 
 # -- Set Breathe parameters and Execute Doxygen -------------------------------
 
+def _dirtree_latest_mtime(dir_path):
+    """
+    Walk a directory and determine the most recent time at which a contained
+    file/directory was modified, created, deleted, removed, etc.
+    """
+
+    # it's important to check the mtime of every directory since they provide
+    # the only indication that files within that directory were deleted/moved
+
+    max_mtime = os.stat(dir_path).st_mtime # need to explicitly check root-dir
+
+    for root, dirs, files in os.walk(dir_path, followlinks = False):
+        for dir in dirs:
+            statinfo = os.stat(os.path.join(root, dir))
+            max_mtime = max(statinfo.st_mtime, max_mtime)
+        for file in files:
+            statinfo = os.stat(os.path.join(root, file))
+            max_mtime = max(statinfo.st_mtime, max_mtime)
+    return int(max_mtime + 1) # gives posix timestamp in seconds (rounded up)
+
 def build_doxygen():
     if os.getenv("SKIPDOXYGEN", "FALSE").lower() == "true":
         # skip a rebuild
         return None
 
+    # load cached modification times (if they exist)
+    try:
+        with open("../cached_mtimes.json", "r") as f:
+            cached_mtimes = json.load(f)
+    except FileNotFoundError:
+        cached_mtimes = {"src" : None, "dox" : None }
+
+    src_mtime = _dirtree_latest_mtime("../../src")
+    if os.path.isdir("../dox-xml"): # skip doxygen if mtimes match cached vals
+        if ((cached_mtimes["src"] == src_mtime) and
+            (cached_mtimes["dox"] == _dirtree_latest_mtime("../dox-xml"))):
+            return None
+
     try:
         retcode = subprocess.call('cd ../../src; doxygen doxygen/Doxyfile-xml',
                                   shell=True)
         if retcode < 0:
             sys.stderr.write("doxygen terminated by signal %s" % (-retcode))
+        else: # cache the md5 hash code for the future
+            with open("../cached_mtimes.json", "w") as f:
+                json.dump({"src" : src_mtime,
+                           "dox" : _dirtree_latest_mtime("../dox-xml")}, f)
     except OSError as e:
         sys.stderr.write("doxygen execution failed: %s" % e)
 

doc/source/index.org

@@ -206,7 +206,7 @@
     of allocations and deallocations are also tracked.  This
     parameter only turns of tracking but not the overloading of
     new()[] and delete()[], which is controlled by the 
-    "memory" variable in the SConstruct file.
+    "memory" variable as a cmake option.
 
 *** TODO Mesh [0/12]
     int num_adapt;

doc/source/param/memory.incl

@@ -6,4 +6,4 @@
 :Default: :d:`true`
 :Scope:     :c:`Cello`
 
-:e:`This parameter is used to turn on or off Cello's build-in memory tracking.  By default it is on, meaning it tracks the number and size of memory allocations, including the current number of bytes allocated, the maximum over the simulation, and the maximum over the current cycle.  Cello implements this by overloading C's new, new[], delete, and delete[] operators.  This can be problematic on some systems, e.g. if an external library also redefines these operators, in which case this parameter should be set to false.  This can be turned off completely by setting "memory = 0" in the top-level "SConstruct" file.`
+:e:`This parameter is used to turn on or off Cello's build-in memory tracking.  By default it is on, meaning it tracks the number and size of memory allocations, including the current number of bytes allocated, the maximum over the simulation, and the maximum over the current cycle.  Cello implements this by overloading C's new, new[], delete, and delete[] operators.  This can be problematic on some systems, e.g. if an external library also redefines these operators, in which case this parameter should be set to false.  This can be turned off completely by setting "memory" OFF (default value) as a cmake option.`

doc/source/param/method.incl

@@ -269,6 +269,16 @@ most up-to-date description of Grackle parameters, see the `Grackle parameters <
 
 ----
 
+:Parameter:  :p:`Method` : :p:`grackle` : :p:`metallicity_floor`
+:Summary: :s:`Minimum metallicity in solar units`
+:Type:    :t:`float`
+:Default: :d:`0.0`
+:Scope:     :z:`Enzo`
+
+:e:`Minimum metallicity in solar units. Must evolve the "metal_density" field in order to use this parameter.`
+
+----
+
 :Parameter:  :p:`Method` : :p:`grackle` : :p:`with_radiative_cooling`
 :Summary:    :s:`Include radiative cooling`
 :Type:       :t:`logical`

doc/source/param/particle.incl

@@ -168,6 +168,21 @@
        }
     }
 
+
+----
+
+:Parameter:  :p:`Particle` : :p:`mass_is_mass`
+:Summary: :s:`Flag for whether particle masses should be handled as a quantity with dimensions of mass.`
+:Type:    :t:`any` 
+:Default: :d:`none`
+:Scope:     :c:`Cello`
+
+This parameter is not used by Enzo-E, but is rather used by yt to indicate whether particle "mass" should be treated as a quantity with dimensions of mass. If this parameter is absent, then
+"mass" is treated as a quantity with dimensions of density, which has a value equal to the true mass divided by the root level cell volume. The value of this parameter is not used: yt only checks
+whether this parameter is present, however, it must have some value which can be read in by yt. It is recommended to set the velue to be "true".
+
+----
+
 -----------
 Performance
 -----------