From 456976cba55256c06e57cb737a84bc2ce8970c0d Mon Sep 17 00:00:00 2001 From: Arjun Barrett Date: Tue, 11 Jun 2024 10:48:07 -0700 Subject: [PATCH] support non-daemonic subprocesses --- python/vmaf/tools/misc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/vmaf/tools/misc.py b/python/vmaf/tools/misc.py index e469045eb..24eacaf5a 100644 --- a/python/vmaf/tools/misc.py +++ b/python/vmaf/tools/misc.py @@ -4,6 +4,7 @@ import unittest from fnmatch import fnmatch import multiprocessing +from concurrent.futures import ProcessPoolExecutor from time import sleep, time import itertools from pathlib import Path @@ -336,8 +337,10 @@ def pool_init(): _pm_list_args = list_args _pm_return_dict = return_dict - with context.Pool(processes, initializer=pool_init) as pool: - pool.map(_parallel_map_rt, range(len(list_args))) + with ProcessPoolExecutor(processes, mp_context=context, initializer=pool_init) as pool: + # ProcessPoolExecutor prevents hanging on the slowest processes that get too much work - delegates one at a time + for _ in pool.map(_parallel_map_rt, range(len(list_args))): + pass return [return_dict[i] for i in range(len(list_args))]