From 1c646e8066621ba8a846310e5759393f85eb8e35 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Mon, 18 Jul 2022 23:21:51 +0200 Subject: [PATCH] postgres --- core/harvester/utils.py | 2 +- core/libs/TasksErrorCodesAnalyser.py | 2 ++ core/libs/task.py | 3 ++- core/pandajob/SQLLookups.py | 4 +++- core/schedresource/utils.py | 34 +++++++++++++++++++++++----- core/views.py | 6 ++--- 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/core/harvester/utils.py b/core/harvester/utils.py index b38d6896b..9cfeeec9f 100644 --- a/core/harvester/utils.py +++ b/core/harvester/utils.py @@ -25,7 +25,7 @@ def isHarvesterJob(pandaid): b.DIAGMESSAGE FROM {settings.DB_SCHEMA_PANDA}.HARVESTER_REL_JOBS_WORKERS a, {settings.DB_SCHEMA_PANDA}.HARVESTER_WORKERS b - WHERE a.harvesterid = b.harvesterid and a.workerid = b.WORKERID) where pandaid = {pandaid} + WHERE a.harvesterid = b.harvesterid and a.workerid = b.WORKERID) tmp_sub where pandaid = {pandaid} """ cur = connection.cursor() cur.execute(sqlQuery) diff --git a/core/libs/TasksErrorCodesAnalyser.py b/core/libs/TasksErrorCodesAnalyser.py index 403bee026..14b970852 100644 --- a/core/libs/TasksErrorCodesAnalyser.py +++ b/core/libs/TasksErrorCodesAnalyser.py @@ -78,6 +78,8 @@ def get_messages_groups(self, tasks_list): tasks_errors_frame = self.remove_stop_words(tasks_errors_frame) tasks_errors_groups = tasks_errors_frame.groupby('processed_errordialog').\ agg({'taskid': lambda x: list(x), 'errordialog': 'first'}).reset_index() + if tasks_errors_groups.empty: + return [] tasks_errors_groups['count'] = tasks_errors_groups.apply(lambda row: len(row['taskid']), axis=1) tasks_errors_groups = tasks_errors_groups.sort_values(by=['count'], ascending=False) diff --git a/core/libs/task.py b/core/libs/task.py index da2d86d07..ed54f48a0 100644 --- a/core/libs/task.py +++ b/core/libs/task.py @@ -360,7 +360,8 @@ def get_task_scouts(jobs): for job in jobs: for jst in scout_types: - if 'jobmetrics' in job and 'scout=' in job['jobmetrics'] and jst in job['jobmetrics'][job['jobmetrics'].index('scout='):]: + if 'jobmetrics' in job and job['jobmetrics'] and 'scout=' in job['jobmetrics'] and \ + jst in job['jobmetrics'][job['jobmetrics'].index('scout='):]: scouts_dict[jst].append(job['pandaid']) # remove scout type if no scouts diff --git a/core/pandajob/SQLLookups.py b/core/pandajob/SQLLookups.py index 0290ca600..9da05992f 100644 --- a/core/pandajob/SQLLookups.py +++ b/core/pandajob/SQLLookups.py @@ -1,5 +1,6 @@ from django.db.models import Transform from django.db.models import Lookup +from django.conf import settings class CastDate(Transform): @@ -9,5 +10,6 @@ class CastDate(Transform): def as_sql(self, compiler, connection): sql, params = compiler.compile(self.lhs) if len(params) > 0: - sql = 'CAST(%s AS DATE)' % sql + if settings.DEPLOYMENT != "POSTGRES": + sql = 'CAST(%s AS DATE)' % sql return sql, params diff --git a/core/schedresource/utils.py b/core/schedresource/utils.py index cf3eed870..f93e9a143 100644 --- a/core/schedresource/utils.py +++ b/core/schedresource/utils.py @@ -1,9 +1,13 @@ """ Utils to get schedresources info from dedicated information system (CRIC) """ +import os + import urllib3 import json import logging +import json +from urllib.parse import urlparse from django.core.cache import cache @@ -19,7 +23,22 @@ def get_CRIC_panda_queues(): if not panda_queues_dict: panda_queues_dict = {} url = settings.CRIC_API_URL - http = urllib3.PoolManager() + # check http proxy + netloc = urlparse(url) + proxy = None + if 'no_proxy' in os.environ and netloc.hostname in os.environ['no_proxy'].split(','): + # no_proxy + pass + elif netloc.scheme == 'https' and 'https_proxy' in os.environ: + # https proxy + proxy = os.environ['https'] + elif netloc.scheme == 'http' and 'http_proxy' in os.environ: + # http proxy + proxy = os.environ['http'] + if proxy: + http = urllib3.ProxyManager(proxy) + else: + http = urllib3.PoolManager() try: r = http.request('GET', url) data = json.loads(r.data.decode('utf-8')) @@ -55,11 +74,14 @@ def get_panda_queues(): panda_queues_list.extend(SchedconfigJson.objects.values()) if len(panda_queues_list) > 0: for pq in panda_queues_list: - try: - panda_queues_dict[pq['pandaqueue']] = json.loads(pq['data']) - except: - panda_queues_dict[pq['pandaqueue']] = None - _logger.error("cannot load json from SCHEDCONFIGJSON table for {} PanDA queue".format(pq['pandaqueue'])) + if isinstance(pq['data'], dict): + panda_queues_dict[pq['pandaqueue']] = pq['data'] + else: + try: + panda_queues_dict[pq['pandaqueue']] = json.loads(pq['data']) + except: + panda_queues_dict[pq['pandaqueue']] = None + _logger.error("cannot load json from SCHEDCONFIGJSON table for {} PanDA queue".format(pq['pandaqueue'])) return panda_queues_dict diff --git a/core/views.py b/core/views.py index 7cdd11159..61863aed2 100644 --- a/core/views.py +++ b/core/views.py @@ -2150,7 +2150,7 @@ def jobInfo(request, pandaid=None, batchid=None, p2=None, p3=None, p4=None): if f['datasetid'] in datasets_dict: f['datasetname'] = datasets_dict[f['datasetid']] - if f['scope'] + ":" in f['datasetname']: + if f['scope'] and f['scope'] + ":" in f['datasetname']: f['ruciodatasetname'] = f['datasetname'].split(":")[1] else: f['ruciodatasetname'] = f['datasetname'] @@ -2159,7 +2159,7 @@ def jobInfo(request, pandaid=None, batchid=None, p2=None, p3=None, p4=None): f['ddmsite'] = panda_queues[job['computingsite']]['gocname'] else: f['ddmsite'] = computeSvsAtlasS.get(job['computingsite'], "") - if 'dst' in f['destinationdblocktoken']: + if f['destinationdblocktoken'] and 'dst' in f['destinationdblocktoken']: parced = f['destinationdblocktoken'].split("_") f['ddmsite'] = parced[0][4:] f['dsttoken'] = parced[1] @@ -2179,7 +2179,7 @@ def jobInfo(request, pandaid=None, batchid=None, p2=None, p3=None, p4=None): if 'creationdate' not in f: f['creationdate'] = f['modificationtime'] if 'fileid' not in f: f['fileid'] = f['row_id'] if 'datasetname' not in f: - if f['scope']+":" in f['dataset']: + if f['scope'] and f['scope']+":" in f['dataset']: f['datasetname'] = f['dataset'] f['ruciodatasetname'] = f['dataset'].split(":")[1] else: