From a6568e289f717e23d9a09912bd4739c75654a421 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 21 Sep 2023 15:32:50 +0200 Subject: [PATCH] Use multiprocessing start method `spawn` by default This is already the default on macOS, and doesn't break anything on Linux. We'll need this at the latest when we want to switch to gojsonnet, since that library doesn't work correctly with multiprocessing start method `fork`. Additionally, we reduce the memory usage of Commodore by using start method `spawn` because the multiprocessing child processes are created with smaller initial memory working sets. However, this causes some execution time overhead because more data needs to be copied to the new processes. Locally, we measured an increase in runtime of approximately 4 seconds when using `spawn` compared to `fork`. --- commodore/cli/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/commodore/cli/__init__.py b/commodore/cli/__init__.py index 1db42b4b..79779c97 100644 --- a/commodore/cli/__init__.py +++ b/commodore/cli/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations +import multiprocessing + from pathlib import Path import click @@ -55,6 +57,8 @@ def commodore(ctx, working_dir, verbose): def main(): + multiprocessing.set_start_method("spawn") + load_dotenv(dotenv_path=find_dotenv(usecwd=True)) commodore.main( prog_name="commodore", auto_envvar_prefix="COMMODORE", max_content_width=100