Merge pull request #541 from NeurodataWithoutBorders/make_default_upl…

…oads_parallel_params_one

Make default upload parallel params 1

pyflask/apis/neuroconv.py

@@ -131,11 +131,9 @@ def post(self):
 
             upload_options = neuroconv_api.payload
             if "number_of_jobs" not in upload_options:
-                upload_options.update(number_of_jobs=psutil.cpu_count(logical=False))
+                upload_options.update(number_of_jobs=1)
             if "number_of_threads" not in upload_options:
-                upload_options.update(
-                    number_of_threads=psutil.cpu_count(logical=True) / psutil.cpu_count(logical=False)
-                )
+                upload_options.update(number_of_threads=1)
 
             return upload_project_to_dandi(**upload_options)
 
@@ -153,11 +151,9 @@ def post(self):
 
             upload_options = neuroconv_api.payload
             if "number_of_jobs" not in upload_options:
-                upload_options.update(number_of_jobs=psutil.cpu_count(logical=False))
+                upload_options.update(number_of_jobs=1)
             if "number_of_threads" not in upload_options:
-                upload_options.update(
-                    number_of_threads=psutil.cpu_count(logical=True) / psutil.cpu_count(logical=False)
-                )
+                upload_options.update(number_of_threads=1)
 
             return upload_folder_to_dandi(**upload_options)
 

schemas/dandi-upload.schema.ts

@@ -6,12 +6,17 @@ import { isStaging } from '../src/renderer/src/stories/pages/uploads/utils'
 import { baseUrl, onServerOpen } from '../src/renderer/src/server/globals'
 import { isStorybook } from '../src/renderer/src/dependencies/simple'
 
+
 const schema = structuredClone(upload)
 const idSchema = schema.properties.dandiset as any
 Object.assign(idSchema, {
     strict: false
 })
 
+const additionalSettings = schema.properties.additional_settings.properties as any
+additionalSettings.number_of_jobs.description = additionalSettings.number_of_threads.description = '<b>⚠️</b> Setting this value to more than 1 has resulted in unpredictable behaviors (runaway processes, segmentation faults, etc.) on several systems</small>'
+
+
 const setReady: any = {}
 
 const createPromise = (prop: string) => new Promise((resolve) => setReady[prop] = resolve)
@@ -26,9 +31,9 @@ onServerOpen(async () => {
     await fetch(new URL("cpus", baseUrl))
     .then((res) => res.json())
     .then(({ physical, logical }) => {
-        const { number_of_jobs, number_of_threads } = schema.properties.additional_settings.properties as any;
-        number_of_jobs.max = number_of_jobs.default = physical;
-        number_of_threads.max = number_of_threads.default = logical / physical;
+        const { number_of_jobs, number_of_threads } = additionalSettings as any;
+        number_of_jobs.max = physical;
+        number_of_threads.max = logical / physical;
         setReady.cpus({ number_of_jobs, number_of_threads })
     })
     .catch(() => {

schemas/json/dandi/upload.json

@@ -9,14 +9,16 @@
                 "number_of_jobs": {
                     "type": "integer",
                     "title": "Job Count",
-                    "description": "The number of files to upload in parallel. <br><small><b>Note:</b> If you encounter an error for any selector value, particularly a Segmentation Fault error, try a value of <code>1</code> to run the jobs without parallelization.</small>",
-                    "min": 1
+                    "description": "The number of files to upload in parallel.",
+                    "min": 1,
+                    "default": 1
                 },
                 "number_of_threads": {
                     "type": "integer",
                     "title": "Threads per Job",
                     "description": "The number of threads to handle each file.",
-                    "min": 1
+                    "min": 1,
+                    "default": 1
                 },
                 "cleanup": {
                     "type": "boolean",

src/renderer/src/stories/pages/uploads/utils.ts

@@ -1,6 +1,7 @@
 
 import { get } from "dandi";
 import dandiUploadSchema from "../../../../../../schemas/dandi-upload.schema";
+import { html } from "lit";
 
 export const isStaging = (id: string) => parseInt(id) >= 100000;
 
@@ -15,6 +16,14 @@ function isNumeric(str: string) {
   export const validate = async (name: string, parent: any) => {
 
     const value = parent[name]
+
+    if (name === 'number_of_jobs' || name === 'number_of_threads') {
+        if (value > 1) return [{
+            type: 'warning',
+            message: `<b>⚠️</b> Increasing the ${name.split('_').join(' ')} may result in unpredictable behaviors.`
+        }]
+    }
+
     if (name === 'dandiset' && value) {
         if (isNumeric(value)){