override cpu limit

catalystneuro · Aug 13, 2024 · d8c9a5f · d8c9a5f
1 parent 3c2a166
commit d8c9a5f
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/src/dandi_s3_log_parser/_command_line_interface.py b/src/dandi_s3_log_parser/_command_line_interface.py
@@ -48,7 +48,7 @@
     "--maximum_number_of_workers",
     help="The maximum number of workers to distribute tasks across.",
     required=False,
-    type=click.IntRange(min=1, max=os.cpu_count()),
+    type=click.IntRange(min=1, max=os.cpu_count() * 5),
     default=1,
 )
 @click.option(

diff --git a/src/dandi_s3_log_parser/_dandi_s3_log_file_parser.py b/src/dandi_s3_log_parser/_dandi_s3_log_file_parser.py
@@ -32,7 +32,7 @@ def parse_all_dandi_raw_s3_logs(
     excluded_log_files: list[FilePath] | None = None,
     excluded_ips: collections.defaultdict[str, bool] | None = None,
     exclude_github_ips: bool = True,
-    maximum_number_of_workers: int = Field(ge=1, le=os.cpu_count(), default=1),
+    maximum_number_of_workers: int = Field(ge=1, le=os.cpu_count() * 5, default=1),
     maximum_buffer_size_in_bytes: int = 4 * 10**9,
 ) -> None:
     """