From bb5916520c6a655cb375e762c82dfe20f810305e Mon Sep 17 00:00:00 2001 From: Tatiana Korchuganova Date: Wed, 26 Apr 2023 17:16:56 +0200 Subject: [PATCH] dash region | ATLASPANDA-809 allow 'Not specified' for resourcetype --- core/dashboards/jobsummaryregion.py | 41 ++++++++++++------- .../templates/JobSummaryRegion.html | 12 +++--- core/pandajob/utils.py | 2 +- core/views.py | 11 +++-- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/core/dashboards/jobsummaryregion.py b/core/dashboards/jobsummaryregion.py index 4e56366f..7411bfd9 100644 --- a/core/dashboards/jobsummaryregion.py +++ b/core/dashboards/jobsummaryregion.py @@ -161,7 +161,7 @@ def get_job_summary_region(query, **kwargs): else: jobtype = 'all' if 'resourcetype' in kwargs and kwargs['resourcetype'] != 'all': - resourcetype = kwargs['jobtype'] + resourcetype = kwargs['resourcetype'] resource_types = [kwargs['resourcetype']] else: resourcetype = 'all' @@ -188,6 +188,31 @@ def get_job_summary_region(query, **kwargs): sites_list = list(set([params['atlas_site'] for pq, params in pqs_dict.items() if 'atlas_site' in params])) regions_list = list(set([params['cloud'] for pq, params in pqs_dict.items() if 'cloud' in params])) + # get job summary + jsq = get_job_summary_split(query, extra=extra) + + # check if there is more values of resourcetype than 4 default ones, if yes -> add them to the list + if len(jsq) > 0: + job_resource_types_extra = list(set([row['resourcetype'] for row in jsq]) - set(resource_types)) + if None in job_resource_types_extra or '' in job_resource_types_extra: + # replace None with 'Not specified' + resource_types.append('Not specified') + for row in jsq: + if row['resourcetype'] is None or row['resourcetype'] == '': + row['resourcetype'] = 'Not specified' + + # adding extra job resourcetype to the list + resource_types.extend([rt for rt in job_resource_types_extra if rt and rt != '']) + + # get workers info + wsq = [] + if 'core.harvester' in settings.INSTALLED_APPS: + from core.harvester.utils import get_workers_summary_split + if 'computingsite__in' not in query: + # put full list of compitingsites to use index in workers table + query['computingsite__in'] = list(set([row['computingsite'] for row in jsq])) + wsq = get_workers_summary_split(query) + # get PanDA getJob, updateJob request counts psq_dict = {} if split_by is None and jobtype == 'all' and resourcetype == 'all': @@ -267,18 +292,6 @@ def get_job_summary_region(query, **kwargs): jsr_sites_dict[s]['all'][rt][em] = 0 jsr_sites_dict[s]['all']['all'][em] = 0 - # get job info - jsq = get_job_summary_split(query, extra=extra) - - # get workers info - wsq = [] - if 'core.harvester' in settings.INSTALLED_APPS: - from core.harvester.utils import get_workers_summary_split - if 'computingsite__in' not in query: - # put full list of compitingsites to use index in workers table - query['computingsite__in'] = list(set([row['computingsite'] for row in jsq])) - wsq = get_workers_summary_split(query) - # fill template with real values of job states counts for row in jsq: if row['computingsite'] in jsr_queues_dict and row['jobtype'] in jsr_queues_dict[row['computingsite']]['summary'] and row['resourcetype'] in resource_types and row['jobstatus'] in const.JOB_STATES and 'count' in row: @@ -368,7 +381,7 @@ def get_job_summary_split(query, extra): select ja4.pandaid, ja4.resource_type, ja4.computingsite, ja4.prodsourcelabel, ja4.jobstatus, ja4.modificationtime, 0 as rcores, case when jobstatus in ('finished', 'failed') then {walltime_sec} else 0 end as walltime, case when transformation like 'http%' then 'run' when transformation like '%.py' then 'py' else 'unknown' end as transform - from {db_scheme}.JOBSARCHIVED4 ja4 where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and {extra_str} + from {db_scheme}.jobsarchived4 ja4 where modificationtime > TO_DATE('{date_from}', 'YYYY-MM-DD HH24:MI:SS') and {extra_str} union select jav4.pandaid, jav4.resource_type, jav4.computingsite, jav4.prodsourcelabel, jav4.jobstatus, jav4.modificationtime, case when jobstatus = 'running' then actualcorecount else 0 end as rcores, 0 as walltime, diff --git a/core/dashboards/templates/JobSummaryRegion.html b/core/dashboards/templates/JobSummaryRegion.html index b45192f8..5f151a24 100644 --- a/core/dashboards/templates/JobSummaryRegion.html +++ b/core/dashboards/templates/JobSummaryRegion.html @@ -283,10 +283,6 @@ ], resourcetype: [ {name:'all', value:'', query: ''}, - {name:'SCORE', value:'SCORE', query: 'resourcetype=SCORE&'}, - {name:'SCORE_HIMEM', value:'SCORE_HIMEM', query: 'resourcetype=SCORE_HIMEM&'}, - {name:'MCORE', value:'MCORE', query: 'resourcetype=MCORE&'}, - {name:'MCORE_HIMEM', value:'MCORE_HIMEM', query: 'resourcetype=MCORE_HIMEM&'} ], }; select_params.region.forEach(function (param) { @@ -294,6 +290,10 @@ $scope.dash.availableOptions.site.push({name:'all', value:'', region:param, query:'' }); }); + select_params.resourcetype.forEach(function (param) { + $scope.dash.availableOptions.resourcetype.push({name:param, value:param, query:'resourcetype' + param + '&'}); + }); + select_params.site.forEach(function (param) { $scope.dash.availableOptions.site.push({name:param.site, value:param.site, region:param.region + '_all', query:'site' + param + '&' }); }); @@ -347,8 +347,8 @@ }; $scope.change = function (selection) { - (selection.jobtype.length > 0) ? $scope.dash.selection.splitby.jobtype = true : $scope.dash.selection.splitby.jobtype = false; - (selection.resourcetype.length > 0) ? $scope.dash.selection.splitby.resourcetype = true : $scope.dash.selection.splitby.resourcetype = false; + (selection.jobtype.length > 3) ? $scope.dash.selection.splitby.jobtype = true : $scope.dash.selection.splitby.jobtype = false; + (selection.resourcetype.length > 3) ? $scope.dash.selection.splitby.resourcetype = true : $scope.dash.selection.splitby.resourcetype = false; } }]); diff --git a/core/pandajob/utils.py b/core/pandajob/utils.py index 48e42b17..faa19e44 100644 --- a/core/pandajob/utils.py +++ b/core/pandajob/utils.py @@ -103,7 +103,7 @@ def identify_jobtype(list_of_dict, field_name='prodsourcelabel'): 'rc_test2': 'analy', } - if settings.DEPLOYMENT == 'ORACLE_DOMA': + if 'ATLAS' not in settings.DEPLOYMENT: psl_to_jt.update({ 'test': 'prod', 'ANY': 'prod', diff --git a/core/views.py b/core/views.py index 6aad8f04..cab1db49 100644 --- a/core/views.py +++ b/core/views.py @@ -871,7 +871,7 @@ def setupView(request, opmode='', hours=0, limit=-99, querytype='job', wildCardE for field in Jobsactive4._meta.get_fields(): if param == field.name: if request.session['requestParams'][param] == 'Not specified': - extraQueryString += " AND ( {0} is NULL or {0} = '' ) ".format(param) + extraQueryString += " AND ( {0} is NULL or {0} = '' ) ".format(field.db_column) extraQueryFields.append(param) continue if param == 'minramcount': @@ -3968,9 +3968,12 @@ def dashRegion(request): split_by=split_by) # prepare lists of unique values for drop down menus - select_params_dict = {} - select_params_dict['queuetype'] = sorted(list(set([pq[1] for pq in jsr_queues_list]))) - select_params_dict['queuestatus'] = sorted(list(set([pq[3] for pq in jsr_queues_list]))) + select_params_dict = { + 'resourcetype': sorted( + [rt for rt in jsr_queues_dict[list(jsr_queues_dict.keys())[0]]['summary']['all'].keys() if rt != 'all']), + 'queuetype': sorted(list(set([pq[1] for pq in jsr_queues_list]))), + 'queuestatus': sorted(list(set([pq[3] for pq in jsr_queues_list]))), + } pq_info_basic = get_basic_info_for_pqs([]) unique_sites_dict = {}