From 6431bea64fbea729f9389d68231e3f83bd7847d3 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Thu, 22 Feb 2024 12:12:33 -0500 Subject: [PATCH 1/3] toasty/par_util.py: if guessing parallelism on Slurm, try to respect job allocation Slurm sets like 20 different environment variables that seem to have to do with specifying the job's CPU allocation. Hopefully this is the right one. --- toasty/par_util.py | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/toasty/par_util.py b/toasty/par_util.py index b8a1481..f411681 100644 --- a/toasty/par_util.py +++ b/toasty/par_util.py @@ -1,16 +1,15 @@ # -*- mode: python; coding: utf-8 -*- -# Copyright 2020 the AAS WorldWide Telescope project +# Copyright 2020-2024 the WorldWide Telescope project # Licensed under the MIT License. """Utilities for parallel processing """ -from __future__ import absolute_import, division, print_function -__all__ = ''' +__all__ = """ SHOW_INFORMATIONAL_MESSAGES resolve_parallelism -'''.split() +""".split() import multiprocessing as mp import os @@ -18,6 +17,7 @@ SHOW_INFORMATIONAL_MESSAGES = True + def resolve_parallelism(parallel): """Decide what level of parallelism to use. @@ -32,17 +32,39 @@ def resolve_parallelism(parallel): """ if parallel is None: - if mp.get_start_method() == 'fork': - parallel = os.cpu_count() + if mp.get_start_method() == "fork": + parallel = None + + # If we seem to be an HPC job, try to guess the number of CPUs based + # on the job allocation of CPUs/cores, which might be much lower + # than the number of CPUs on the system. Slurm sets many variables + # related to this stuff; I *think* the one we're using here is the + # most appropriate? + + slurm_alloc = os.environ.get("SLURM_NPROCS") + if slurm_alloc: + try: + parallel = int(slurm_alloc) + except ValueError: + pass + + # If we're still not sure, go with the system CPU count + + if parallel is None: + parallel = os.cpu_count() + if SHOW_INFORMATIONAL_MESSAGES and parallel > 1: - print(f'info: parallelizing processing over {parallel} CPUs') + print(f"info: parallelizing processing over {parallel} CPUs") else: parallel = 1 - if parallel > 1 and mp.get_start_method() != 'fork': - print('''warning: parallel processing was requested but is not possible - because this operating system is not using `fork`-based multiprocessing - On macOS a bug prevents forking: https://bugs.python.org/issue33725''', file=sys.stderr) + if parallel > 1 and mp.get_start_method() != "fork": + print( + """warning: parallel processing was requested but is not possible + because this operating system is not using `fork`-based multiprocessing. + On macOS a bug prevents forking: https://bugs.python.org/issue33725""", + file=sys.stderr, + ) parallel = 1 if parallel > 1: From 19fdec350b080551266a29c7f32c2918658d8a8d Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Sat, 20 Jul 2024 10:32:48 -0400 Subject: [PATCH 2/3] Update for Numpy 2.0 compatibility --- pyproject.toml | 12 ++---------- setup.py | 6 +++++- toasty/_libtoasty.pyx | 4 +++- toasty/samplers.py | 3 ++- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dd523fb..c74d140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,6 @@ [build-system] -requires = [ - 'cython', - 'oldest-supported-numpy', - 'setuptools', - 'wheel', -] +requires = ['cython', 'numpy', 'setuptools', 'wheel'] build-backend = 'setuptools.build_meta' [tool.cranko] -annotated_files = [ - "docs/conf.py", - "toasty/cli.py", -] +annotated_files = ["docs/conf.py", "toasty/cli.py"] diff --git a/setup.py b/setup.py index aed36bc..516ba05 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,11 @@ def get_long_desc(): "build_ext": build_ext, }, ext_modules=[ - Extension("toasty._libtoasty", ["toasty/_libtoasty.pyx"]), + Extension( + "toasty._libtoasty", + ["toasty/_libtoasty.pyx"], + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], + ), ], include_dirs=[ np.get_include(), diff --git a/toasty/_libtoasty.pyx b/toasty/_libtoasty.pyx index 70e4dad..762a0aa 100644 --- a/toasty/_libtoasty.pyx +++ b/toasty/_libtoasty.pyx @@ -1,9 +1,11 @@ from libc.math cimport sin, cos, atan2, hypot -import numpy as np cimport cython cimport numpy as np +import numpy as np + +np.import_array() DTYPE = np.float64 ctypedef np.float64_t DTYPE_t diff --git a/toasty/samplers.py b/toasty/samplers.py index b68e633..e5c7492 100644 --- a/toasty/samplers.py +++ b/toasty/samplers.py @@ -152,7 +152,8 @@ def healpix_fits_file_sampler( # needed. data = data[data.dtype.names[0]] if data.dtype.byteorder not in "=|": - data = data.byteswap().newbyteorder() + data = data.byteswap() + data = data.view(data.dtype.newbyteorder()) nest = hdr.get("ORDERING") == "NESTED" coord = hdr.get("COORDSYS", "C") From a61c814a75a9f8ccfc2a95b4d1272bd5b66544d8 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Sun, 21 Jul 2024 11:29:52 -0400 Subject: [PATCH 3/3] ci/azure-build-and-test.yml: update Mac build image --- ci/azure-build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure-build-and-test.yml b/ci/azure-build-and-test.yml index fde849f..cb8de6e 100644 --- a/ci/azure-build-and-test.yml +++ b/ci/azure-build-and-test.yml @@ -16,12 +16,12 @@ parameters: PYTHON_SERIES: "3.9" - name: macos_310 - vmImage: macos-11 + vmImage: macos-12 vars: PYTHON_SERIES: "3.10" - name: macos_39 - vmImage: macos-11 + vmImage: macos-12 vars: PYTHON_SERIES: "3.9"