PM job state management (#3702)

* better pm job state management

* move job search to function

* use job_details_params

apps/dashboard/app/controllers/projects_controller.rb

@@ -103,12 +103,13 @@ def destroy
 
   # GET /projects/:project_id/jobs/:cluster/:jobid
   def job_details
+    project = Project.find(job_details_params[:project_id])
     cluster_str = job_details_params[:cluster].to_s
     cluster = OodAppkit.clusters[cluster_str.to_sym]
     render(:status => 404) if cluster.nil?
 
-    job_info = cluster.job_adapter.info(job_details_params[:jobid].to_s)
-    hpc_job = HpcJob.from_core_info(info: job_info, cluster: cluster_str)
+    hpc_job = project.job_from_id(job_details_params[:jobid].to_s, cluster_str)
+
     render(partial: 'job_details', locals: { job: hpc_job })
   end
 

apps/dashboard/app/models/launcher.rb

@@ -209,6 +209,15 @@ def jobs
     end
   end
 
+  # When a job is requested, update before returning
+  def job_from_id(job_id, cluster)
+    job = stored_job_from_id(job_id, cluster)
+    unless job.nil? || job.status.to_s == 'completed'
+      update_job_log(job_id, cluster)
+    end
+    stored_job_from_id(job_id, cluster)
+  end
+
   def create_default_script
     return false if Launcher.scripts?(project_dir) || default_script_path.exist?
 
@@ -307,11 +316,23 @@ def most_recent_job
     end.reverse.first.to_h
   end
 
+  def stored_job_from_id(job_id, cluster)
+    jobs.detect { |j| j.id == job_id && j.cluster == cluster }
+  end
+
   def update_job_log(job_id, cluster)
     adapter = adapter(cluster).job_adapter
     info = adapter.info(job_id)
     job = HpcJob.from_core_info(info: info, cluster: cluster)
-    new_jobs = (jobs + [job.to_h]).map(&:to_h)
+    existing_jobs = jobs
+    stored_job = stored_job_from_id(job_id, cluster)
+    if stored_job.nil?
+      new_jobs = (jobs + [job.to_h]).map(&:to_h)
+    else
+      new_jobs = existing_jobs.map(&:to_h)
+      idx = existing_jobs.index(stored_job)
+      new_jobs[idx].merge!(job.to_h) { |key, old_val, new_val| new_val.nil? ? old_val : new_val } 
+    end
 
     File.write(job_log_file.to_s, new_jobs.to_yaml)
   end

apps/dashboard/app/models/project.rb

@@ -203,6 +203,14 @@ def jobs
     end.flatten
   end
 
+  def job_from_id(job_id, cluster)
+    launchers = Launcher.all(directory)
+    launchers.each do |launcher|
+      job = launcher.job_from_id(job_id, cluster)
+      return job unless job.nil?
+    end
+  end
+
   def readme_path
     file = Dir.glob("#{directory}/README.{md,txt}").first.to_s
     File.readable?(file) ? file : nil