From f81af8b16eaa9813234017ed8e624c244b54f44a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:36:22 -0500 Subject: [PATCH 1/5] make default upload parallel params 1 --- schemas/json/dandi/upload.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index e74ac92bb..269b07020 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -10,13 +10,15 @@ "type": "integer", "title": "Job Count", "description": "The number of files to upload in parallel.
Note: If you encounter an error for any selector value, particularly a Segmentation Fault error, try a value of 1 to run the jobs without parallelization.", - "min": 1 + "min": 1, + "default": 1 }, "number_of_threads": { "type": "integer", "title": "Threads per Job", "description": "The number of threads to handle each file.", - "min": 1 + "min": 1, + "default": 1 }, "cleanup": { "type": "boolean", From 32cd2d2003bbe4bbbabfd065fb8d54322e7d4b7e Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 13 Dec 2023 14:52:41 -0800 Subject: [PATCH 2/5] Add note and finish disabling the cpu defaults --- pyflask/apis/neuroconv.py | 8 ++++---- schemas/dandi-upload.schema.ts | 4 ++-- schemas/json/dandi/upload.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 9cf10ec0e..aea618647 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -131,10 +131,10 @@ def post(self): upload_options = neuroconv_api.payload if "number_of_jobs" not in upload_options: - upload_options.update(number_of_jobs=psutil.cpu_count(logical=False)) + upload_options.update(number_of_jobs=1) if "number_of_threads" not in upload_options: upload_options.update( - number_of_threads=psutil.cpu_count(logical=True) / psutil.cpu_count(logical=False) + number_of_threads=1 ) return upload_project_to_dandi(**upload_options) @@ -153,10 +153,10 @@ def post(self): upload_options = neuroconv_api.payload if "number_of_jobs" not in upload_options: - upload_options.update(number_of_jobs=psutil.cpu_count(logical=False)) + upload_options.update(number_of_jobs=1) if "number_of_threads" not in upload_options: upload_options.update( - number_of_threads=psutil.cpu_count(logical=True) / psutil.cpu_count(logical=False) + number_of_threads=1 ) return upload_folder_to_dandi(**upload_options) diff --git a/schemas/dandi-upload.schema.ts b/schemas/dandi-upload.schema.ts index 59729eced..d62fd5ae6 100644 --- a/schemas/dandi-upload.schema.ts +++ b/schemas/dandi-upload.schema.ts @@ -27,8 +27,8 @@ onServerOpen(async () => { .then((res) => res.json()) .then(({ physical, logical }) => { const { number_of_jobs, number_of_threads } = schema.properties.additional_settings.properties as any; - number_of_jobs.max = number_of_jobs.default = physical; - number_of_threads.max = number_of_threads.default = logical / physical; + number_of_jobs.max = physical; + number_of_threads.max = logical / physical; setReady.cpus({ number_of_jobs, number_of_threads }) }) .catch(() => { diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index 269b07020..360ac5a0d 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -9,7 +9,7 @@ "number_of_jobs": { "type": "integer", "title": "Job Count", - "description": "The number of files to upload in parallel.
Note: If you encounter an error for any selector value, particularly a Segmentation Fault error, try a value of 1 to run the jobs without parallelization.", + "description": "The number of files to upload in parallel.
⚠️ Use With Caution: Setting this value above one has resulted in unpreditable behaviors (runaway processes, segmentation errors, etc.) on several systems", "min": 1, "default": 1 }, From c560909e725c63de7772880391b9a32089d249d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 22:54:13 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/neuroconv.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index aea618647..0c9154a79 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -133,9 +133,7 @@ def post(self): if "number_of_jobs" not in upload_options: upload_options.update(number_of_jobs=1) if "number_of_threads" not in upload_options: - upload_options.update( - number_of_threads=1 - ) + upload_options.update(number_of_threads=1) return upload_project_to_dandi(**upload_options) @@ -155,9 +153,7 @@ def post(self): if "number_of_jobs" not in upload_options: upload_options.update(number_of_jobs=1) if "number_of_threads" not in upload_options: - upload_options.update( - number_of_threads=1 - ) + upload_options.update(number_of_threads=1) return upload_folder_to_dandi(**upload_options) From b29bc0aa981c8756d0999461b8d0c7b5fc506c14 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 13 Dec 2023 15:03:06 -0800 Subject: [PATCH 4/5] Include a warning for both number_of_x properties --- schemas/dandi-upload.schema.ts | 7 ++++++- schemas/json/dandi/upload.json | 2 +- src/renderer/src/stories/pages/uploads/utils.ts | 9 +++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/schemas/dandi-upload.schema.ts b/schemas/dandi-upload.schema.ts index d62fd5ae6..50f109f1d 100644 --- a/schemas/dandi-upload.schema.ts +++ b/schemas/dandi-upload.schema.ts @@ -6,12 +6,17 @@ import { isStaging } from '../src/renderer/src/stories/pages/uploads/utils' import { baseUrl, onServerOpen } from '../src/renderer/src/server/globals' import { isStorybook } from '../src/renderer/src/dependencies/simple' + const schema = structuredClone(upload) const idSchema = schema.properties.dandiset as any Object.assign(idSchema, { strict: false }) +const additionalSettings = schema.properties.additional_settings.properties as any +additionalSettings.number_of_jobs.description = additionalSettings.number_of_threads.description = '⚠️ Setting this value has resulted in unpreditable behaviors (runaway processes, segmentation errors, etc.) on several systems' + + const setReady: any = {} const createPromise = (prop: string) => new Promise((resolve) => setReady[prop] = resolve) @@ -26,7 +31,7 @@ onServerOpen(async () => { await fetch(new URL("cpus", baseUrl)) .then((res) => res.json()) .then(({ physical, logical }) => { - const { number_of_jobs, number_of_threads } = schema.properties.additional_settings.properties as any; + const { number_of_jobs, number_of_threads } = additionalSettings as any; number_of_jobs.max = physical; number_of_threads.max = logical / physical; setReady.cpus({ number_of_jobs, number_of_threads }) diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index 360ac5a0d..ced194f5e 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -9,7 +9,7 @@ "number_of_jobs": { "type": "integer", "title": "Job Count", - "description": "The number of files to upload in parallel.
⚠️ Use With Caution: Setting this value above one has resulted in unpreditable behaviors (runaway processes, segmentation errors, etc.) on several systems", + "description": "The number of files to upload in parallel.", "min": 1, "default": 1 }, diff --git a/src/renderer/src/stories/pages/uploads/utils.ts b/src/renderer/src/stories/pages/uploads/utils.ts index 806348d1b..1cdff2822 100644 --- a/src/renderer/src/stories/pages/uploads/utils.ts +++ b/src/renderer/src/stories/pages/uploads/utils.ts @@ -1,6 +1,7 @@ import { get } from "dandi"; import dandiUploadSchema from "../../../../../../schemas/dandi-upload.schema"; +import { html } from "lit"; export const isStaging = (id: string) => parseInt(id) >= 100000; @@ -15,6 +16,14 @@ function isNumeric(str: string) { export const validate = async (name: string, parent: any) => { const value = parent[name] + + if (name === 'number_of_jobs' || name === 'number_of_threads') { + if (value > 1) return [{ + type: 'warning', + message: `⚠️ Increasing the ${name.split('_').join(' ')} may result in unpredictable behaviors.` + }] + } + if (name === 'dandiset' && value) { if (isNumeric(value)){ From 7fc92158eeb7685026b04ce67d341a026d773790 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:29:57 -0500 Subject: [PATCH 5/5] Update schemas/dandi-upload.schema.ts --- schemas/dandi-upload.schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/dandi-upload.schema.ts b/schemas/dandi-upload.schema.ts index 50f109f1d..49f82623d 100644 --- a/schemas/dandi-upload.schema.ts +++ b/schemas/dandi-upload.schema.ts @@ -14,7 +14,7 @@ Object.assign(idSchema, { }) const additionalSettings = schema.properties.additional_settings.properties as any -additionalSettings.number_of_jobs.description = additionalSettings.number_of_threads.description = '⚠️ Setting this value has resulted in unpreditable behaviors (runaway processes, segmentation errors, etc.) on several systems' +additionalSettings.number_of_jobs.description = additionalSettings.number_of_threads.description = '⚠️ Setting this value to more than 1 has resulted in unpredictable behaviors (runaway processes, segmentation faults, etc.) on several systems' const setReady: any = {}