Change default log directory

lxm3/xm_cluster/artifacts.py

@@ -38,6 +38,9 @@ def job_path(self, job_name: str):
     def job_script_path(self, job_name: str):
         return os.path.join(self.job_path(job_name), "job.sh")
 
+    def job_log_path(self, job_name: str):
+        return os.path.join(os.path.join(self._storage_root, "logs", job_name))
+
     def singularity_image_path(self, image_name: str):
         return os.path.join(self._storage_root, "containers", image_name)
 
@@ -80,7 +83,7 @@ def _put_file(self, local_filename, dst):
 
     def deploy_job_scripts(self, job_name, job_script):
         job_path = self.job_path(job_name)
-        job_log_path = os.path.join(job_path, "logs")
+        job_log_path = self.job_log_path(job_name)
 
         self._fs.makedirs(job_path, exist_ok=True)
         self._fs.makedirs(job_log_path, exist_ok=True)

lxm3/xm_cluster/execution/gridengine.py

@@ -58,11 +58,11 @@ def _create_job_header(
         cls,
         executor: executors.GridEngine,
         num_array_tasks: Optional[int],
-        job_script_dir: str,
+        job_log_dir: str,
         job_name: str,
     ) -> str:
         job_header = header_from_executor(
-            job_name, executor, num_array_tasks, job_script_dir
+            job_name, executor, num_array_tasks, job_log_dir
         )
         return job_header
 
@@ -170,7 +170,7 @@ def header_from_executor(
     job_name: str,
     executor: executors.GridEngine,
     num_array_tasks: Optional[int],
-    job_script_dir: str,
+    job_log_dir: str,
 ) -> str:
     header = []
 
@@ -210,7 +210,7 @@ def header_from_executor(
     if reserved:
         header.append("#$ -R y")
 
-    log_directory = executor.log_directory or os.path.join(job_script_dir, "logs")
+    log_directory = executor.log_directory or job_log_dir
     if num_array_tasks is not None:
         stdout = os.path.join(log_directory, "$JOB_NAME.o$JOB_ID.$TASK_ID")
         stderr = os.path.join(log_directory, "$JOB_NAME.e$JOB_ID.$TASK_ID")

lxm3/xm_cluster/execution/job_script.py

@@ -305,7 +305,7 @@ def _create_job_header(
         cls,
         executor: xm.Executor,
         num_array_tasks: Optional[int],
-        job_script_dir: str,
+        job_log_dir: str,
         job_name: str,
     ) -> str:
         """Create a job header"""
@@ -408,15 +408,15 @@ def build(
         self,
         job: Union[xm.Job, array_job.ArrayJob],
         job_name: str,
-        job_script_dir: str,
+        job_log_dir: str,
     ) -> str:
         setup = self._create_setup_cmds(job.executable, job.executor)
 
         num_array_tasks = None
         if isinstance(job, array_job.ArrayJob) and len(job.args) > 1:
             num_array_tasks = len(job.args)
         header = self._create_job_header(
-            job.executor, num_array_tasks, job_script_dir, job_name
+            job.executor, num_array_tasks, job_log_dir, job_name
         )
 
         return self._create_job_script(job=job, setup=setup, header=header)
@@ -434,8 +434,9 @@ def _launch(self, job_script_path: str, num_jobs: int) -> Tuple[Optional[str], A
     def launch(self, job_name: str, job: ClusterJob):
         job_name = re.sub("\\W", "_", job_name)
         job_script_dir = self._artifact_store.job_path(job_name)
+        job_log_dir = self._artifact_store.job_log_path(job_name)
         job_script_builder = self.builder_cls(self._settings)
-        job_script_content = job_script_builder.build(job, job_name, job_script_dir)
+        job_script_content = job_script_builder.build(job, job_name, job_log_dir)
 
         self._artifact_store.deploy_job_scripts(job_name, job_script_content)
         job_script_path = os.path.join(job_script_dir, JOB_SCRIPT_NAME)

lxm3/xm_cluster/execution/local.py

@@ -62,10 +62,10 @@ def _create_job_header(
         cls,
         executor: executors.GridEngine,
         num_array_tasks: Optional[int],
-        job_script_dir: str,
+        job_log_dir: str,
         job_name: str,
     ) -> str:
-        del executor, num_array_tasks, job_script_dir, job_name
+        del executor, num_array_tasks, job_log_dir, job_name
         return ""
 
     def build(

lxm3/xm_cluster/execution/slurm.py

@@ -47,12 +47,12 @@ def _create_job_header(
         cls,
         executor: executors.Slurm,
         num_array_tasks: Optional[int],
-        job_script_dir: str,
+        job_log_dir: str,
         job_name: str,
     ) -> str:
         num_array_tasks = None
         job_header = header_from_executor(
-            job_name, executor, num_array_tasks, job_script_dir
+            job_name, executor, num_array_tasks, job_log_dir
         )
         return job_header
 
@@ -172,7 +172,7 @@ def header_from_executor(
     job_name: str,
     executor: executors.Slurm,
     num_array_tasks: Optional[int],
-    job_script_dir: str,
+    job_log_dir: str,
 ) -> str:
     header = []
 
@@ -188,7 +188,7 @@ def header_from_executor(
         duration = executor.walltime
         header.append(f"#SBATCH --time={_format_slurm_time(duration)}")
 
-    log_directory = executor.log_directory or os.path.join(job_script_dir, "logs")
+    log_directory = executor.log_directory or job_log_dir
     if num_array_tasks is not None:
         stdout = os.path.join(log_directory, "slurm-%A_%a.out")
     else: