Max mem workers (#29)

Apply a maximum of 60GB to executor memory.
OSC · Apr 14, 2023 · 46aac3a · 46aac3a
1 parent bc33a44
commit 46aac3a
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/template/before.sh.erb b/template/before.sh.erb
@@ -31,6 +31,11 @@
     end
   end
 
+  def exec_memory
+    calculated = avail_mem/context.num_workers.to_i
+    [60, calculated].min
+  end
+
   spark_config = {
     "spark.ui.reverseProxy" => "true",
     #"spark.ui.reverseProxyUrl" => "https://ondemand.osc.edu/node/${SPARK_MASTER_HOST}/${SPARK_MASTER_WEBUI_PORT}",
@@ -46,7 +51,7 @@
 
   def pyspark_submit_args
     args = []
-    args.concat(["--executor-memory  #{avail_mem/context.num_workers.to_i}G"]) unless extra_spark_config.has_key?('spark.executor.memory')
+    args.concat(["--executor-memory  #{exec_memory}G"]) unless extra_spark_config.has_key?('spark.executor.memory')
     args.concat(['--conf spark.driver.maxResultSize=0']) unless extra_spark_config.has_key?("spark.driver.maxResultSize")
 
     if extra_spark_config.has_key?('spark.driver.memory')