Merge pull request #8 from ucb-rit/fix_parsing_sinfo

fix `sinfo` parsing and two issues with handling reservations

sq.py

@@ -23,10 +23,10 @@ def freeze(dest_dir, name, data):
     with open(path.join(dest_dir, name), 'w') as f:
         f.write(data)
 
-def read_slurm_as_df(stdout):
+def read_slurm_as_df(stdout, sep = '|'):
     # The first line of the data is the column headers, the rest are job data entries
     # Rows are separated by newline, columns are separated by pipe |
-    columns, *data = [ row.split('|') for row in stdout.strip().split('\n') ]
+    columns, *data = [ row.split(sep) for row in stdout.strip().split('\n') ]
 
     # Sometimes the fields can contain the pipe symbol |
     # In this case it is impossible to parse unambiguously so the row is thrown out
@@ -96,10 +96,11 @@ def get_assoc_df(stdout):
 
 @load_from('sinfo')
 def sinfo_cmd():
-    return run_cmd(['sinfo', '--format', '%all'])
+    return run_cmd(['sinfo', '-N', '--format', '%N,%P'])
+    ## return run_cmd(['sinfo', '--format', '%all'])
 
 def get_sinfo_df(stdout):
-    df = read_slurm_as_df(stdout)
+    df = read_slurm_as_df(stdout, ',')
     df.columns = df.columns.str.strip()
     return df
 
@@ -365,7 +366,7 @@ def check_resv_conflicts(pending_job, resv_df, sinfo_df):
         non_resv_nodes = non_resv_nodes - set(resv['Nodes'])
         if delta:
             general_interfering_resvs.append(resv)
-    if len(non_resv_nodes) >= remaining_num_nodes:
+    if len(non_resv_nodes) < remaining_num_nodes:
         interfering_resvs += general_interfering_resvs
 
     return interfering_resvs
@@ -432,7 +433,7 @@ def inner(pending_job):
                 f"This job is requesting: {job_resources_requested}",
                 suggest_other_qos(slurm_info, pending_job)
             ])
-        if pending_job['REASON'] in ['Priority', 'Resources']:
+        if pending_job['REASON'] in ['Priority', 'Resources'] or re.match("^ReqNodeNotAvail", pending_job['REASON']):
             sprio_df = slurm_info.sprio_df()
             resv_conflicts = check_resv_conflicts(pending_job, slurm_info.resv_df(), slurm_info.sinfo_df())
             if resv_conflicts: