Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added limit to the number of parallel pipelines #560

Open
wants to merge 1 commit into
base: future
Choose a base branch
from
Open

added limit to the number of parallel pipelines #560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions compiler/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ def add_common_arguments(parser):
help="Run multiple pipelines in parallel if they are safe to run",
action="store_true",
default=False)
parser.add_argument("--parallel_pipelines_limit",
angelhof marked this conversation as resolved.
Show resolved Hide resolved
type=int,
help="Configure the limit for the number of parallel pipelines at one time (default: cpu count, 0 is turns parallel pipelines off)",
default=os.cpu_count())
parser.add_argument("--r_split",
help="(experimental) use round robin split, merge, wrap, and unwrap",
action="store_true")
Expand Down Expand Up @@ -197,6 +201,8 @@ def pass_common_arguments(pash_arguments):
arguments.append(string_to_argument("--distributed_exec"))
if (pash_arguments.parallel_pipelines):
arguments.append(string_to_argument("--parallel_pipelines"))
arguments.append(string_to_argument("--parallel_pipelines_limit"))
arguments.append(string_to_argument(str(pash_arguments.parallel_pipelines_limit)))
if (pash_arguments.daemon_communicates_through_unix_pipes):
arguments.append(string_to_argument("--daemon_communicates_through_unix_pipes"))
arguments.append(string_to_argument("--r_split_batch_size"))
Expand Down
11 changes: 8 additions & 3 deletions compiler/pash_runtime_daemon.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,7 @@ def compile_and_add(self, compiled_script_file, var_file, input_ir_file):
self.wait_for_all()

if compile_success:
self.wait_for_limit(config.pash_args.parallel_pipelines_limit)
response = success_response(
f'{process_id} {compiled_script_file} {var_file} {input_ir_file}')
else:
Expand Down Expand Up @@ -357,16 +358,20 @@ def get_next_id(self):
self.next_id += 1
return self.next_id

def wait_for_all(self):
log("Waiting for all processes to finish. There are", self.running_procs, "processes remaining.")
while self.running_procs > 0:
def wait_for_limit(self, limit):
log("Waiting for for number of processes to be less than limit")
while self.running_procs > limit:
input_cmd = self.get_input()
# must be exit command or something is wrong
if (input_cmd.startswith("Exit:")):
self.handle_exit(input_cmd)
else:
raise Exception(
f"Command should be exit but it was {input_cmd}")

def wait_for_all(self):
log("Waiting for all processes to finish. There are", self.running_procs, "processes remaining.")
self.wait_for_limit(0)
self.unsafe_running = False

def handle_exit(self, input_cmd):
Expand Down