Merge branch 'master' into v4-prep

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.3.1
+current_version = 3.4.0
 commit = False
 tag = False
 

.github/workflows/test_suite.yaml

@@ -65,7 +65,7 @@ jobs:
  if: "contains(env.commit_message, 'ci debug')"
  run: |
  echo "log_level=ULTRA_DEBUG" >> $GITHUB_ENV
- echo "extra_pytest_args=-s -k "test_power_spectra_lightcone[mdz_tsfluct_nthreads]" --log-level-21=DEBUG" >> $GITHUB_ENV
+ echo "extra_pytest_args=-s --log-level-21=DEBUG" >> $GITHUB_ENV
 
  - name: Make it a Normal Run
  if: "!contains(env.commit_message, 'ci debug')"
@@ -99,7 +99,7 @@ jobs:
  python -m pytest -n 2 -l ${{ env.extra_pytest_args }} --cov=py21cmfast --cov-config=.coveragerc -vv --cov-report xml:./coverage.xml --durations=25 --plots=testplots
 
  - name: Archive Integration Test Plots
- if: always() && contains(env.commit_message, 'ci debug')
+ if: always()
  uses: actions/upload-artifact@v3
  with:
  name: integration-test-plots-${{ matrix.os }}-${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
  - id: mixed-line-ending
  args: ['--fix=no']
  - repo: https://github.com/pycqa/flake8
- rev: 7.0.0 # pick a git hash / tag to point to
+ rev: 7.1.1 # pick a git hash / tag to point to
  hooks:
  - id: flake8
  additional_dependencies:
@@ -34,7 +34,7 @@ repos:
  - flake8-print
 
  - repo: https://github.com/psf/black-pre-commit-mirror
- rev: 24.4.2
+ rev: 24.8.0
  hooks:
  - id: black
 
@@ -51,7 +51,7 @@ repos:
  - id: rst-inline-touching-normal
 
  - repo: https://github.com/asottile/pyupgrade
- rev: v3.16.0
+ rev: v3.17.0
  hooks:
  - id: pyupgrade
  args: [--py36-plus]

.readthedocs.yml

@@ -10,9 +10,9 @@ sphinx:
  configuration: docs/conf.py
 
 build:
- os: ubuntu-22.04
+ os: ubuntu-24.04
  tools:
- python: mambaforge-22.9
+ python: mambaforge-latest
 
 conda:
  environment: docs/environment.yaml

CHANGELOG.rst

@@ -4,6 +4,14 @@ Changelog
 dev-version
 -----------
 
+v3.4.0 [07 Aug 2024]
+----------------------
+
+Performance
+~~~~~~~~~~~
+
+* When reading in data from cache, only read the necessary fields.
+
 Deprecations
 ~~~~~~~~~~~~
 
@@ -13,6 +21,9 @@ Fixed
 ~~~~~
 
 * Fixed small issue in ``Lightcone.lightcone_coords``.
+* Bug when setting ``NON_CUBIC_FACTOR != 1``.
+* Using new ``matplotlib`` versions and registering ``cmap``.
+* Small bug in lightcone interpolation.
 
 Added
 ~~~~~
@@ -44,6 +55,10 @@ Fixed
 ~~~~~
 
 * Incorrect sign on adiabatic fluctuations.
+* Spurious warnings about boxes being read.
+* Spurious warnings about configuration files being out of date.
+* Compilation for ``clang`` and newer ``gcc`` versions.
+
 
 v3.3.1 [24 May 2023]
 ----------------------

VERSION

@@ -1 +1 @@
-3.3.1
+3.4.0

ci/macos-latest-env.yml

@@ -18,7 +18,7 @@ dependencies:
  - gsl
  - fftw
  - pip:
- - powerbox
+ - powerbox<0.8.1
  - cached_property
  - pre-commit
  - pytest-plt

ci/ubuntu-latest-env.yml

@@ -16,7 +16,7 @@ dependencies:
  - matplotlib
  - pip
  - pip:
- - powerbox
+ - powerbox<0.8.1
  - cached_property
  - pre-commit
  - pytest-plt

setup.py

@@ -51,6 +51,7 @@ def _read(*names, **kwargs):
  ),
  re.sub(":[a-z]+:`~?(.*?)`", r"``\1``", _read("CHANGELOG.rst")),
  ),
+ long_description_content_type="text/x-rst",
  author="The 21cmFAST coredev team",
  author_email="[email protected]",
  url="https://github.com/21cmFAST/21cmFAST",

src/py21cmfast/_cfg.py

@@ -39,7 +39,10 @@ def __init__(self, *args, write=True, file_name=None, **kwargs):
  for k, v in self._defaults.items():
  if k not in self:
  if k not in self._aliases:
- warnings.warn("Your configuration file is out of date. Updating...")
+ if self.file_name:
+ warnings.warn(
+ "Your configuration file is out of date. Updating..."
+ )
  do_write = True
  self[k] = v
 
@@ -55,9 +58,10 @@ def __init__(self, *args, write=True, file_name=None, **kwargs):
  del self[alias]
  break
  else:
- warnings.warn(
- "Your configuration file is out of date. Updating..."
- )
+ if self.file_name:
+ warnings.warn(
+ "Your configuration file is out of date. Updating..."
+ )
  do_write = True
  self[k] = v
 
@@ -70,7 +74,10 @@ def __init__(self, *args, write=True, file_name=None, **kwargs):
  self["direc"] = Path(self["direc"]).expanduser().absolute()
 
  if do_write and write and self.file_name:
- self.write()
+ try:
+ self.write()
+ except Exception:
+ pass
 
  @contextlib.contextmanager
  def use(self, **kwargs):
@@ -84,8 +91,7 @@ def use(self, **kwargs):
 
  def write(self, fname: str | Path | None = None):
  """Write current configuration to file to make it permanent."""
- fname = Path(fname or self.file_name)
- if fname:
+ if fname := Path(fname or self.file_name):
  if not fname.parent.exists():
  fname.parent.mkdir(parents=True)
 
@@ -106,7 +112,7 @@ def load(cls, file_name: str | Path):
  cfg = yaml.load(fl)
  return cls(cfg, file_name=file_name)
  else:
- return cls(write=False)
+ return cls(write=True)
 
 
 config = Config.load(Path("~/.21cmfast/config.yml"))

src/py21cmfast/cache_tools.py

@@ -181,10 +181,16 @@ def query_cache(
  for file in list_datasets(
  direc=direc, kind=kind, hsh=hsh, seed=seed, redshift=redshift
  ):
- cls = readbox(direc=direc, fname=file, load_data=False)
+ try:
+ kls = readbox(direc=direc, fname=file, load_data=False)
+ except OSError as e:
+ warnings.warn(f"Failed to read {file}: {e}")
+ continue
+
  if show:
- print(file + ": " + str(cls)) # noqa: T
- yield file, cls
+ print(f"{file}: {str(kls)}") # noqa: T201
+
+ yield file, kls
 
 
 def get_boxes_at_redshift(
@@ -200,7 +206,6 @@ def get_boxes_at_redshift(
  obj = readbox(direc=direc, fname=file, load_data=False)
  except OSError:
  warnings.warn(f"Failed to read {file}")
- pass
 
  if not hasattr(obj, "redshift"):
  logger.debug(f"{file} has no redshift")

src/py21cmfast/cli.py

@@ -603,22 +603,41 @@ def lightcone(ctx, redshift, config, out, regen, direc, max_z, seed, lq):
  print(f"Saved Lightcone to {fname}.")
 
 
-def _query(direc=None, kind=None, md5=None, seed=None, clear=False):
- cls = list(
+def _query(
+ direc=None,
+ kind=None,
+ md5=None,
+ seed=None,
+ clear=False,
+ sort_by=("kind", "redshift"),
+ show=None,
+):
+ objects = list(
  cache_tools.query_cache(direc=direc, kind=kind, hsh=md5, seed=seed, show=False)
  )
 
  if not clear:
- print("%s Data Sets Found:" % len(cls))
+ print(f"{len(objects)} Data Sets Found:")
  print("------------------")
  else:
- print("Removing %s data sets..." % len(cls))
+ print(f"Removing {len(objects)} data sets...")
 
- for file, c in cls:
- if not clear:
- print(" @ {%s}:" % file)
- print(" %s" % str(c))
+ for sorter in sort_by[::-1]:
+ if sorter == "kind":
+ objects = sorted(objects, key=lambda x: x[1].__class__.__name__)
+ elif sorter == "filename":
+ objects = sorted(objects, key=lambda x: x[0])
+ else:
+ objects = sorted(objects, key=lambda x: getattr(x[1], sorter, 0.0))
 
+ for file, c in objects:
+ if not clear:
+ print(f" @ {file}:")
+ if not show:
+ print(f" {c}")
+ else:
+ for s in show:
+ print(f" {s}: {getattr(c, s, None)}")
  print()
 
  else:
@@ -642,7 +661,9 @@ def _query(direc=None, kind=None, md5=None, seed=None, clear=False):
  default=False,
  help="remove all data sets returned by this query.",
 )
-def query(direc, kind, md5, seed, clear):
+@click.option("--fields", type=str, multiple=True, default=None)
+@click.option("--sort", type=str, multiple=True, default=["kind", "redshift"])
+def query(direc, kind, md5, seed, clear, fields, sort):
  """Query the cache database.
 
  Parameters
@@ -658,7 +679,7 @@ def query(direc, kind, md5, seed, clear):
  clear : bool
  Remove all data sets returned by the query.
  """
- _query(direc, kind, md5, seed, clear)
+ _query(direc, kind, md5, seed, clear, show=fields, sort_by=tuple(sort))
 
 
 @main.command()

src/py21cmfast/src/BrightnessTemperatureBox.c

@@ -5,7 +5,7 @@ void get_velocity_gradient(struct UserParams *user_params, float *v, float *vel_
 {
  memcpy(vel_gradient, v, sizeof(fftwf_complex)*HII_KSPACE_NUM_PIXELS);
 
- dft_r2c_cube(user_params->USE_FFTW_WISDOM, user_params->HII_DIM, HII_D_PARA, user_params->N_THREADS, vel_gradient);
+ dft_r2c_cube(user_params->USE_FFTW_WISDOM, user_params->HII_DIM, HII_D_PARA, user_params->N_THREADS, (fftwf_complex *)vel_gradient);
 
  float k_x, k_y, k_z;
  int n_x, n_y, n_z;
@@ -25,7 +25,7 @@ void get_velocity_gradient(struct UserParams *user_params, float *v, float *vel_
  k_y = n_y * DELTA_K;
 
  for (n_z=0; n_z<=HII_MIDDLE_PARA; n_z++){
- k_z = n_z * DELTA_K;
+ k_z = n_z * DELTA_K_PARA;
 
  // take partial deriavative along the line of sight
  *((fftwf_complex *) vel_gradient + HII_C_INDEX(n_x,n_y,n_z)) *= k_z*I/(float)HII_TOT_NUM_PIXELS;
@@ -34,7 +34,7 @@ void get_velocity_gradient(struct UserParams *user_params, float *v, float *vel_
  }
  }
 
- dft_c2r_cube(user_params->USE_FFTW_WISDOM, user_params->HII_DIM, HII_D_PARA, user_params->N_THREADS, vel_gradient);
+ dft_c2r_cube(user_params->USE_FFTW_WISDOM, user_params->HII_DIM, HII_D_PARA, user_params->N_THREADS, (fftwf_complex *)vel_gradient);
 }
 
 int ComputeBrightnessTemp(float redshift, struct UserParams *user_params, struct CosmoParams *cosmo_params,

src/py21cmfast/src/PerturbField.c

@@ -88,7 +88,7 @@ void compute_perturbed_velocities(
  }
 
  LOG_SUPER_DEBUG("density_perturb after modification by dDdt: ");
- debugSummarizeBox(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
 
  if(user_params->PERTURB_ON_HIGH_RES) {
 
@@ -508,9 +508,9 @@ int ComputePerturbField(
 
  LOG_SUPER_DEBUG("density_perturb: ");
  if(user_params->PERTURB_ON_HIGH_RES){
- debugSummarizeBox(HIRES_density_perturb, dimension, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(HIRES_density_perturb, dimension, user_params->NON_CUBIC_FACTOR, " ");
  }else{
- debugSummarizeBox(LOWRES_density_perturb, dimension, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(LOWRES_density_perturb, dimension, user_params->NON_CUBIC_FACTOR, " ");
  }
 
  // deallocate
@@ -621,7 +621,7 @@ int ComputePerturbField(
  }
 
  LOG_SUPER_DEBUG("LOWRES_density_perturb: ");
- debugSummarizeBox(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
 
  // transform to k-space
  dft_r2c_cube(user_params->USE_FFTW_WISDOM, user_params->HII_DIM, HII_D_PARA, user_params->N_THREADS, LOWRES_density_perturb);
@@ -632,15 +632,15 @@ int ComputePerturbField(
  }
 
  LOG_SUPER_DEBUG("LOWRES_density_perturb after smoothing: ");
- debugSummarizeBox(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
 
  // save a copy of the k-space density field
  memcpy(LOWRES_density_perturb_saved, LOWRES_density_perturb, sizeof(fftwf_complex)*HII_KSPACE_NUM_PIXELS);
 
  dft_c2r_cube(user_params->USE_FFTW_WISDOM, user_params->HII_DIM, HII_D_PARA, user_params->N_THREADS, LOWRES_density_perturb);
 
  LOG_SUPER_DEBUG("LOWRES_density_perturb back in real space: ");
- debugSummarizeBox(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
 
  // normalize after FFT
  int bad_count=0;
@@ -665,7 +665,7 @@ int ComputePerturbField(
  }
  if(bad_count>=5) LOG_WARNING("Total number of bad indices for LOW_density_perturb: %d", bad_count);
  LOG_SUPER_DEBUG("LOWRES_density_perturb back in real space (normalized): ");
- debugSummarizeBox(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
+ debugSummarizeBoxComplex(LOWRES_density_perturb, user_params->HII_DIM, user_params->NON_CUBIC_FACTOR, " ");
 
 
 #pragma omp parallel shared(perturbed_field,LOWRES_density_perturb) private(i,j,k) num_threads(user_params->N_THREADS)