Merge pull request #417 from 21cmfast/c-restructure

C restructure

.github/workflows/test_suite_nointegration.yaml

@@ -0,0 +1,113 @@
+name: Tests (No Integration Tests)
+
+# Test on all pushes, except when the push is literally just a tag (because we
+# tag automatically via CI, and therefore there's no extra code in that push).
+# Also, only test on pull requests into master/production.
+on:
+ push:
+ tags-ignore:
+ - 'v*'
+ pull_request:
+ branches:
+ - 'v4-prep'
+
+
+jobs:
+ tests:
+ if: "!contains(github.event.pull_request.labels.*.name, 'auto-pr')"
+ env:
+ ENV_NAME: tests
+ PYTHON: ${{ matrix.python-version }}
+ OS: ${{ matrix.os }}
+ CC: gcc
+ name: Testing
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest]
+ python-version: [3.9, "3.10", "3.11", "3.12"]
+ defaults:
+ run:
+ # Adding -l {0} ensures conda can be found properly in each step
+ shell: bash -l {0}
+ steps:
+ - uses: actions/checkout@master
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ fetch-depth: 0
+
+ - name: Print head git commit message
+ id: get_head_commit_message
+ run: echo "commit_message=$(git show -s --format=%s)" >> "$GITHUB_ENV"
+
+ - name: Setup Miniconda
+ uses: conda-incubator/setup-miniconda@v3
+ with:
+ # auto-update-conda: true
+ mamba-version: "*"
+ channels: conda-forge,defaults
+ python-version: ${{ matrix.python-version }}
+ environment-file: ci/${{ matrix.os }}-env.yml
+ activate-environment: tests
+ channel-priority: true
+
+ - name: Conda Info
+ run: |
+ conda info -a
+ conda list
+ conda config --show-sources
+ conda config --show
+ printenv | sort
+
+ - name: Make it a Debug Run
+ if: "contains(env.commit_message, 'ci debug')"
+ run: |
+ echo "log_level=ULTRA_DEBUG" >> $GITHUB_ENV
+ echo "extra_pytest_args=-s --log-level-21=DEBUG" >> $GITHUB_ENV
+
+ - name: Make it a Normal Run
+ if: "!contains(env.commit_message, 'ci debug')"
+ run: |
+ echo "log_level=INFO" >> $GITHUB_ENV
+ echo "extra_pytest_args=" >> $GITHUB_ENV
+
+ - name: Get C Libraries Linux
+ if: matrix.os == 'ubuntu-latest'
+ run: |
+ sudo apt-get install libfftw3-dev
+ sudo apt-get install libgsl0-dev
+
+ - name: Setup GCC
+ uses: Dup4/actions-setup-gcc@v1
+ with:
+ version: latest
+
+ - name: Install 21cmFAST Linux
+ if: matrix.os == 'ubuntu-latest'
+ run: |
+ LOG_LEVEL=${{ env.log_level }} pip install .
+
+ - name: Install 21cmFAST MacOS
+ if: matrix.os == 'macos-latest'
+ run: |
+ LOG_LEVEL=${{ env.log_level }} CFLAGS="-isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk" pip install .
+
+ - name: Run Tests
+ run: |
+ python -m pytest -n 2 -l ${{ env.extra_pytest_args }} --ignore tests/test_integration_features.py --cov=py21cmfast --cov-config=.coveragerc -vv --cov-report xml:./coverage.xml --durations=25 --plots=testplots
+
+ - name: Archive Integration Test Plots
+ if: always()
+ uses: actions/upload-artifact@v3
+ with:
+ name: integration-test-plots-${{ matrix.os }}-${{ matrix.python-version }}
+ path: |
+ testplots/*.pdf
+
+ - uses: codecov/codecov-action@v3
+ if: matrix.os == 'ubuntu-latest' && success() && !contains(github.event.pull_request.labels.*.name, 'auto-pr')
+ with:
+ fail_ci_if_error: true
+ verbose: true
+ token: ${{ secrets.CODECOV_TOKEN }}

build_cffi.py

@@ -21,28 +21,11 @@
 CLOC = os.path.join(LOCATION, "src", "py21cmfast", "src")
 include_dirs = [CLOC]
 
-# ==================================================
-# Set compilation arguments dependent on environment
-# ==================================================
-extra_compile_args = ["-w", "--verbose"]
-
-if "DEBUG" in os.environ:
- extra_compile_args += ["-g", "-O0"]
-else:
- extra_compile_args += ["-Ofast"]
-
-if sys.platform == "darwin":
- extra_compile_args += ["-Xpreprocessor"]
-
-extra_compile_args += ["-fopenmp"]
-
-libraries = ["m", "gsl", "gslcblas", "fftw3f_omp", "fftw3f"]
-
-# stuff for gperftools
-if "PROFILE" in os.environ:
- libraries += ["profiler", "tcmalloc"]
- # we need this even if DEBUG is off
- extra_compile_args += ["-g"]
+c_files = [
+ os.path.join("src", "py21cmfast", "src", f)
+ for f in os.listdir(CLOC)
+ if f.endswith(".c")
+]
 
 # Set the C-code logging level.
 # If DEBUG is set, we default to the highest level, but if not,
@@ -71,42 +54,79 @@
  "of {}".format(available_levels)
  )
 
+# ==================================================
+# Set compilation arguments dependent on environment
+# ==================================================
+
+extra_compile_args = ["-Wall", "--verbose", f"-DLOG_LEVEL={log_level:d}"]
+
+if "DEBUG" in os.environ:
+ extra_compile_args += ["-g", "-O0"]
+else:
+ extra_compile_args += ["-Ofast"]
+
+if sys.platform == "darwin":
+ extra_compile_args += ["-Xpreprocessor"]
+
+extra_compile_args += ["-fopenmp"]
+
+libraries = ["m", "gsl", "gslcblas", "fftw3f_omp", "fftw3f"]
+
+# stuff for gperftools
+if "PROFILE" in os.environ:
+ libraries += ["profiler", "tcmalloc"]
+ # we need this even if DEBUG is off
+ extra_compile_args += ["-g"]
+
+if compiler == "clang":
+ libraries += ["omp"]
+
 library_dirs = []
 for k, v in os.environ.items():
  if "inc" in k.lower():
  include_dirs += [v]
  elif "lib" in k.lower():
  library_dirs += [v]
 
-libraries = ["m", "gsl", "gslcblas", "fftw3f_omp", "fftw3f"]
-
-if compiler == "clang":
- libraries += ["omp"]
-
 # =================================================================
+# NOTES FOR DEVELOPERS:
+# The CFFI implementation works as follows:
+# - All function prototypes, global variables and type definitions *directly* used
+# in the python wrapper must be declared via ffi.cdef("""C CODE""").
+# There must be no compiler directives in this code (#include, #define, etc)
+# - All implementations of global variables and types present in the cdef() calls
+# must also be present in the second argument of set_source.
+# This is passed to the compiler.
+# - The `sources` kwarg then contains all the .c files in the library which are to be compiled
 
 # This is the overall C code.
 ffi.set_source(
  "py21cmfast.c_21cmfast", # Name/Location of shared library module
  """
- #define LOG_LEVEL {log_level}
-
- #include "GenerateICs.c"
- """.format(
- log_level=log_level
- ),
+ #include "21cmFAST.h"
+ """,
+ sources=c_files,
  include_dirs=include_dirs,
  library_dirs=library_dirs,
  libraries=libraries,
  extra_compile_args=extra_compile_args,
 )
 
-# This is the Header file
-with open(os.path.join(CLOC, "21cmFAST.h")) as f:
+# Header files containing types, globals and function prototypes
+with open(os.path.join(CLOC, "_inputparams_wrapper.h")) as f:
  ffi.cdef(f.read())
-
-with open(os.path.join(CLOC, "Globals.h")) as f:
+with open(os.path.join(CLOC, "_outputstructs_wrapper.h")) as f:
  ffi.cdef(f.read())
+with open(os.path.join(CLOC, "_functionprototypes_wrapper.h")) as f:
+ ffi.cdef(f.read())
+
+# CFFI needs to be able to access a free function to make the __del__ method for OutputStruct fields
+# This will expose the standard free() function to the wrapper so it can be used
+ffi.cdef(
+ """
+ void free(void *ptr);
+ """
+)
 
 if __name__ == "__main__":
  ffi.compile()

src/py21cmfast/_utils.py

@@ -289,13 +289,13 @@ def _cstruct(self):
 
  def _new(self):
  """Return a new empty C structure corresponding to this class."""
- return self._ffi.new("struct " + self._name + "*")
+ return self._ffi.new(self._name + "*")
 
  @classmethod
  def get_fields(cls, cstruct=None) -> Dict[str, Any]:
  """Obtain the C-side fields of this struct."""
  if cstruct is None:
- cstruct = cls._ffi.new("struct " + cls._get_name() + "*")
+ cstruct = cls._ffi.new(cls._get_name() + "*")
  return dict(cls._ffi.typeof(cstruct[0]).fields)
 
  @classmethod

src/py21cmfast/outputs.py

@@ -328,7 +328,7 @@ def __init__(
  self,
  *,
  desc_redshift: float | None = None,
- buffer_size: int = 0.0,
+ buffer_size: int = 0,
  **kwargs,
  ):
  self.desc_redshift = desc_redshift

src/py21cmfast/photoncons.py

@@ -58,11 +58,11 @@
  lib.adjust_redshifts_for_photoncons()
  lib.determine_deltaz_for_photoncons() (first time only) --> calculates the deltaz array with some smoothing
  --> does more smoothing and returns the adjusted redshift
- ELIF PHOTON_CONS_TYPE==1:
+ ELIF PHOTON_CONS_TYPE==2:
  photoncons_alpha() --> computes and stores ALPHA_ESC shift vs global neutral fraction
  lib.ComputeIonizedBox()
  get_fesc_fit() --> applies the fit to the current redshift
- ELIF PHOTON_CONS_TYPE==2:
+ ELIF PHOTON_CONS_TYPE==3:
  photoncons_fesc() --> computes and stores F_ESC10 shift vs global neutral fraction
  lib.ComputeIonizedBox()
  get_fesc_fit() --> applies the fit to the current redshift
@@ -327,11 +327,13 @@ def calibrate_photon_cons(
  astro_params_photoncons = deepcopy(astro_params)
  astro_params_photoncons._R_BUBBLE_MAX = astro_params.R_BUBBLE_MAX
 
- flag_options_photoncons = FlagOptions(
- USE_MASS_DEPENDENT_ZETA=flag_options.USE_MASS_DEPENDENT_ZETA,
- M_MIN_in_Mass=flag_options.M_MIN_in_Mass,
+ flag_options_photoncons = flag_options.clone(
+ USE_TS_FLUCT=False,
+ INHOMO_RECO=False,
+ USE_MINI_HALOS=False,
+ HALO_STOCHASTICITY=False,
+ PHOTON_CONS_TYPE=0,
  )
-
  ib = None
  prev_perturb = None