make spark.task.cpus check more robust so that it doesn't crash local…

… tests
saalfeldlab · Sep 10, 2023 · 5eb67cd · 5eb67cd
1 parent a806c6c
commit 5eb67cd
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/.../main/java/org/janelia/render/client/spark/intensityadjust/IntensityCorrectionClient.java b/.../main/java/org/janelia/render/client/spark/intensityadjust/IntensityCorrectionClient.java
@@ -78,7 +78,13 @@ public void runWithContext(final JavaSparkContext sparkContext)
 
         // allow multithread tasks but warn if Spark isn't configured properly to support them
         final int threadsForSparkTasks = cmdLineSetup.distributedSolve.threadsWorker;
-        final String taskCpusString = sparkContext.getConf().get("spark.task.cpus");
+        String taskCpusString = null;
+        try {
+            final SparkConf sparkConf = sparkContext.getConf();
+            taskCpusString = sparkConf.get("spark.task.cpus");
+        } catch (final Exception e) {
+            LOG.warn("ignoring failure to retrieve spark.task.cpus value", e);
+        }
         if (taskCpusString != null) {
             final int taskCpus = Integer.parseInt(taskCpusString);
             if (taskCpus != threadsForSparkTasks) {