Merge pull request #164 from PanDAWMS/dev

dash region | ATLASPANDA-809 allow 'Not specified' for resourcetype

core/dashboards/jobsummaryregion.py

@@ -161,7 +161,7 @@ def get_job_summary_region(query, **kwargs):
     else:
         jobtype = 'all'
     if 'resourcetype' in kwargs and kwargs['resourcetype'] != 'all':
-        resourcetype = kwargs['jobtype']
+        resourcetype = kwargs['resourcetype']
         resource_types = [kwargs['resourcetype']]
     else:
         resourcetype = 'all'
@@ -188,6 +188,31 @@ def get_job_summary_region(query, **kwargs):
     sites_list = list(set([params['atlas_site'] for pq, params in pqs_dict.items() if 'atlas_site' in params]))
     regions_list = list(set([params['cloud'] for pq, params in pqs_dict.items() if 'cloud' in params]))
 
+    # get job summary
+    jsq = get_job_summary_split(query, extra=extra)
+
+    # check if there is more values of resourcetype than 4 default ones, if yes -> add them to the list
+    if len(jsq) > 0:
+        job_resource_types_extra = list(set([row['resourcetype'] for row in jsq]) - set(resource_types))
+        if None in job_resource_types_extra or '' in job_resource_types_extra:
+            # replace None with 'Not specified'
+            resource_types.append('Not specified')
+            for row in jsq:
+                if row['resourcetype'] is None or row['resourcetype'] == '':
+                    row['resourcetype'] = 'Not specified'
+
+        # adding extra job resourcetype to the list
+        resource_types.extend([rt for rt in job_resource_types_extra if rt and rt != ''])
+
+    # get workers info
+    wsq = []
+    if 'core.harvester' in settings.INSTALLED_APPS:
+        from core.harvester.utils import get_workers_summary_split
+        if 'computingsite__in' not in query:
+            # put full list of compitingsites to use index in workers table
+            query['computingsite__in'] = list(set([row['computingsite'] for row in jsq]))
+        wsq = get_workers_summary_split(query)
+
     # get PanDA getJob, updateJob request counts
     psq_dict = {}
     if split_by is None and jobtype == 'all' and resourcetype == 'all':
@@ -267,18 +292,6 @@ def get_job_summary_region(query, **kwargs):
                     jsr_sites_dict[s]['all'][rt][em] = 0
                     jsr_sites_dict[s]['all']['all'][em] = 0
 
-    # get job info
-    jsq = get_job_summary_split(query, extra=extra)
-
-    # get workers info
-    wsq = []
-    if 'core.harvester' in settings.INSTALLED_APPS:
-        from core.harvester.utils import get_workers_summary_split
-        if 'computingsite__in' not in query:
-            # put full list of compitingsites to use index in workers table
-            query['computingsite__in'] = list(set([row['computingsite'] for row in jsq]))
-        wsq = get_workers_summary_split(query)
-
     # fill template with real values of job states counts
     for row in jsq:
         if row['computingsite'] in jsr_queues_dict and row['jobtype'] in jsr_queues_dict[row['computingsite']]['summary'] and row['resourcetype'] in resource_types and row['jobstatus'] in const.JOB_STATES and 'count' in row:
@@ -368,7 +381,7 @@ def get_job_summary_split(query, extra):
         select ja4.pandaid, ja4.resource_type, ja4.computingsite, ja4.prodsourcelabel, ja4.jobstatus, ja4.modificationtime, 0 as rcores,
             case when jobstatus in ('finished', 'failed') then {walltime_sec} else 0 end as walltime,
             case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform
-        from {db_scheme}.JOBSARCHIVED4 ja4  where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and {extra_str}
+        from {db_scheme}.jobsarchived4 ja4  where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and {extra_str}
         union
         select jav4.pandaid, jav4.resource_type, jav4.computingsite, jav4.prodsourcelabel, jav4.jobstatus, jav4.modificationtime,
             case when jobstatus = 'running' then actualcorecount else 0 end as rcores, 0 as walltime,

core/dashboards/templates/JobSummaryRegion.html

@@ -283,17 +283,17 @@
         ],
         resourcetype: [
           {name:'all', value:'', query: ''},
-          {name:'SCORE', value:'SCORE', query: 'resourcetype=SCORE&'},
-          {name:'SCORE_HIMEM', value:'SCORE_HIMEM', query: 'resourcetype=SCORE_HIMEM&'},
-          {name:'MCORE', value:'MCORE', query: 'resourcetype=MCORE&'},
-          {name:'MCORE_HIMEM', value:'MCORE_HIMEM', query: 'resourcetype=MCORE_HIMEM&'}
         ],
     };
     select_params.region.forEach(function (param) {
      $scope.dash.availableOptions.region.push({name:param, value:param, query:'region' + param + '&'});
      $scope.dash.availableOptions.site.push({name:'all', value:'', region:param, query:'' });
     });
 
+    select_params.resourcetype.forEach(function (param) {
+     $scope.dash.availableOptions.resourcetype.push({name:param, value:param, query:'resourcetype' + param + '&'});
+    });
+
     select_params.site.forEach(function (param) {
      $scope.dash.availableOptions.site.push({name:param.site, value:param.site, region:param.region + '_all', query:'site' + param + '&' });
     });
@@ -347,8 +347,8 @@
     };
 
     $scope.change = function (selection) {
-        (selection.jobtype.length > 0) ? $scope.dash.selection.splitby.jobtype = true : $scope.dash.selection.splitby.jobtype = false;
-        (selection.resourcetype.length > 0) ? $scope.dash.selection.splitby.resourcetype = true : $scope.dash.selection.splitby.resourcetype = false;
+        (selection.jobtype.length > 3) ? $scope.dash.selection.splitby.jobtype = true : $scope.dash.selection.splitby.jobtype = false;
+        (selection.resourcetype.length > 3) ? $scope.dash.selection.splitby.resourcetype = true : $scope.dash.selection.splitby.resourcetype = false;
     }
 }]);
 

core/pandajob/utils.py

@@ -103,7 +103,7 @@ def identify_jobtype(list_of_dict, field_name='prodsourcelabel'):
         'rc_test2': 'analy',
     }
 
-    if settings.DEPLOYMENT == 'ORACLE_DOMA':
+    if 'ATLAS' not in settings.DEPLOYMENT:
         psl_to_jt.update({
             'test': 'prod',
             'ANY': 'prod',

core/views.py

@@ -871,7 +871,7 @@ def setupView(request, opmode='', hours=0, limit=-99, querytype='job', wildCardE
             for field in Jobsactive4._meta.get_fields():
                 if param == field.name:
                     if request.session['requestParams'][param] == 'Not specified':
-                        extraQueryString += " AND ( {0} is NULL or {0} = '' ) ".format(param)
+                        extraQueryString += " AND ( {0} is NULL or {0} = '' ) ".format(field.db_column)
                         extraQueryFields.append(param)
                         continue
                     if param == 'minramcount':
@@ -3968,9 +3968,12 @@ def dashRegion(request):
             split_by=split_by)
 
         # prepare lists of unique values for drop down menus
-        select_params_dict = {}
-        select_params_dict['queuetype'] = sorted(list(set([pq[1] for pq in jsr_queues_list])))
-        select_params_dict['queuestatus'] = sorted(list(set([pq[3] for pq in jsr_queues_list])))
+        select_params_dict = {
+            'resourcetype': sorted(
+                [rt for rt in jsr_queues_dict[list(jsr_queues_dict.keys())[0]]['summary']['all'].keys() if rt != 'all']),
+            'queuetype': sorted(list(set([pq[1] for pq in jsr_queues_list]))),
+            'queuestatus': sorted(list(set([pq[3] for pq in jsr_queues_list]))),
+        }
 
         pq_info_basic = get_basic_info_for_pqs([])
         unique_sites_dict = {}