From 9be0a56ddce51fdb87f7b9a9c87637de5266c010 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 2 Nov 2023 08:39:10 -0400 Subject: [PATCH 1/4] warn that fork should not be used for macos --- doc/modules/core.rst | 6 +++--- src/spikeinterface/core/job_tools.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/modules/core.rst b/doc/modules/core.rst index 976a82a4a3..4c03950b1d 100644 --- a/doc/modules/core.rst +++ b/doc/modules/core.rst @@ -518,9 +518,9 @@ These are a set of keyword arguments which are common to all functions that supp A float like 0.5 means half of the availables core. * progress_bar: bool If True, a progress bar is printed -* mp_context: str or None - Context for multiprocessing. It can be None (default), "fork" or "spawn". - Note that "fork" is only available on UNIX systems (not Windows) +* mp_context: "fork" | "spawn" | None, default: None + "fork" or "spawn". If None, the context is taken by the recording.get_preferred_mp_context(). + "fork" is only safely available on LINUX systems. The default **job_kwargs** are :code:`n_jobs=1, chunk_duration="1s", progress_bar=True`. diff --git a/src/spikeinterface/core/job_tools.py b/src/spikeinterface/core/job_tools.py index a9df4f4626..3c6106084a 100644 --- a/src/spikeinterface/core/job_tools.py +++ b/src/spikeinterface/core/job_tools.py @@ -5,6 +5,7 @@ import numpy as np import platform import os +import warnings import joblib import sys @@ -30,9 +31,9 @@ Number of jobs to use. With -1 the number of jobs is the same as number of cores * progress_bar: bool If True, a progress bar is printed - * mp_context: str or None - Context for multiprocessing. It can be None (default), "fork" or "spawn". - Note that "fork" is only available on UNIX systems + * mp_context: "fork" | "spawn" | None, default: None + Context for multiprocessing. It can be None, "fork" or "spawn". + Note that "fork" is only safely available on LINUX systems """ @@ -287,9 +288,9 @@ class ChunkRecordingExecutor: Size of each chunk in number of samples. If "total_memory" or "chunk_memory" are used, it is ignored. chunk_duration : str or float or None Chunk duration in s if float or with units if str (e.g. "1s", "500ms") - mp_context : str or None, default: None + mp_context : "fork" | "spawn" | None, default: None "fork" or "spawn". If None, the context is taken by the recording.get_preferred_mp_context(). - "fork" is only available on UNIX systems. + "fork" is only safely available on LINUX systems. max_threads_per_process: int or None, default: None Limit the number of thread per process using threadpoolctl modules. This used only when n_jobs>1 @@ -332,6 +333,8 @@ def __init__( mp_context = recording.get_preferred_mp_context() if mp_context is not None and platform.system() == "Windows": assert mp_context != "fork", "'fork' mp_context not supported on Windows!" + elif mp_context is not None and platform.system() == "Darwin": + warnings.warn('As of Python 3.8 "fork" is no longer considered safe on MacOS') self.mp_context = mp_context From 22a7b79ca23162da3be326bece24b69801a1847f Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 2 Nov 2023 08:44:10 -0400 Subject: [PATCH 2/4] fix typo in installation tips --- .github/workflows/installation-tips-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/installation-tips-test.yml b/.github/workflows/installation-tips-test.yml index d4529bdaff..ce4a9d16ec 100644 --- a/.github/workflows/installation-tips-test.yml +++ b/.github/workflows/installation-tips-test.yml @@ -36,4 +36,4 @@ jobs: run: python ./installation_tips/check_your_install.py --ci # ci flag turns off gui - name: Windows cleanup if: ${{ matrix.label == 'windows' }} - run: python .installation_tips/cleanup_for_windows.py + run: python ./installation_tips/cleanup_for_windows.py From 3e2f41196e0ba083805aa32079e58e80fb29ebed Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:28:36 -0400 Subject: [PATCH 3/4] add general warning to use of fork, revert installation tips since Alessio has PR fix --- .github/workflows/installation-tips-test.yml | 2 +- src/spikeinterface/core/job_tools.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/installation-tips-test.yml b/.github/workflows/installation-tips-test.yml index ce4a9d16ec..d4529bdaff 100644 --- a/.github/workflows/installation-tips-test.yml +++ b/.github/workflows/installation-tips-test.yml @@ -36,4 +36,4 @@ jobs: run: python ./installation_tips/check_your_install.py --ci # ci flag turns off gui - name: Windows cleanup if: ${{ matrix.label == 'windows' }} - run: python ./installation_tips/cleanup_for_windows.py + run: python .installation_tips/cleanup_for_windows.py diff --git a/src/spikeinterface/core/job_tools.py b/src/spikeinterface/core/job_tools.py index 3c6106084a..f94490986f 100644 --- a/src/spikeinterface/core/job_tools.py +++ b/src/spikeinterface/core/job_tools.py @@ -333,8 +333,10 @@ def __init__( mp_context = recording.get_preferred_mp_context() if mp_context is not None and platform.system() == "Windows": assert mp_context != "fork", "'fork' mp_context not supported on Windows!" - elif mp_context is not None and platform.system() == "Darwin": - warnings.warn('As of Python 3.8 "fork" is no longer considered safe on MacOS') + elif mp_context == "fork" and platform.system() == "Darwin": + warnings.warn('As of Python 3.8 "fork" is no longer considered safe on macOS') + elif mp_context == "fork": + warnings.warn('Use of "fork" will be deprecated in Python. Consider "spawn" in the future') self.mp_context = mp_context From 8bc3b61af1e05a29856a435a8ce847d24da407f8 Mon Sep 17 00:00:00 2001 From: zm711 <92116279+zm711@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:06:25 -0400 Subject: [PATCH 4/4] remove linux warning --- src/spikeinterface/core/job_tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/spikeinterface/core/job_tools.py b/src/spikeinterface/core/job_tools.py index f94490986f..fcf1d93d1c 100644 --- a/src/spikeinterface/core/job_tools.py +++ b/src/spikeinterface/core/job_tools.py @@ -335,8 +335,6 @@ def __init__( assert mp_context != "fork", "'fork' mp_context not supported on Windows!" elif mp_context == "fork" and platform.system() == "Darwin": warnings.warn('As of Python 3.8 "fork" is no longer considered safe on macOS') - elif mp_context == "fork": - warnings.warn('Use of "fork" will be deprecated in Python. Consider "spawn" in the future') self.mp_context = mp_context