Merge pull request #132 from tmaeno/psg

to fix pandamon@postgres

core/harvester/utils.py

@@ -25,7 +25,7 @@ def isHarvesterJob(pandaid):
       b.DIAGMESSAGE
       FROM {settings.DB_SCHEMA_PANDA}.HARVESTER_REL_JOBS_WORKERS a,
       {settings.DB_SCHEMA_PANDA}.HARVESTER_WORKERS b
-      WHERE a.harvesterid = b.harvesterid and a.workerid = b.WORKERID) where pandaid = {pandaid}
+      WHERE a.harvesterid = b.harvesterid and a.workerid = b.WORKERID) tmp_sub where pandaid = {pandaid}
   """
     cur = connection.cursor()
     cur.execute(sqlQuery)

core/libs/TasksErrorCodesAnalyser.py

@@ -78,6 +78,8 @@ def get_messages_groups(self, tasks_list):
             tasks_errors_frame = self.remove_stop_words(tasks_errors_frame)
             tasks_errors_groups = tasks_errors_frame.groupby('processed_errordialog').\
                 agg({'taskid': lambda x: list(x), 'errordialog': 'first'}).reset_index()
+            if tasks_errors_groups.empty:
+                return []
             tasks_errors_groups['count'] = tasks_errors_groups.apply(lambda row: len(row['taskid']), axis=1)
             tasks_errors_groups = tasks_errors_groups.sort_values(by=['count'], ascending=False)
 

core/libs/task.py

@@ -360,7 +360,8 @@ def get_task_scouts(jobs):
 
     for job in jobs:
         for jst in scout_types:
-            if 'jobmetrics' in job and 'scout=' in job['jobmetrics'] and jst in job['jobmetrics'][job['jobmetrics'].index('scout='):]:
+            if 'jobmetrics' in job and job['jobmetrics'] and 'scout=' in job['jobmetrics'] and \
+                    jst in job['jobmetrics'][job['jobmetrics'].index('scout='):]:
                 scouts_dict[jst].append(job['pandaid'])
 
     # remove scout type if no scouts

core/pandajob/SQLLookups.py

@@ -1,5 +1,6 @@
 from django.db.models import Transform
 from django.db.models import Lookup
+from django.conf import settings
 
 
 class CastDate(Transform):
@@ -9,5 +10,6 @@ class CastDate(Transform):
     def as_sql(self, compiler, connection):
         sql, params = compiler.compile(self.lhs)
         if len(params) > 0:
-            sql = 'CAST(%s AS DATE)' % sql
+            if settings.DEPLOYMENT != "POSTGRES":
+                sql = 'CAST(%s AS DATE)' % sql
         return sql, params

core/schedresource/utils.py

@@ -1,9 +1,13 @@
 """
 Utils to get schedresources info from dedicated information system (CRIC)
 """
+import os
+
 import urllib3
 import json
 import logging
+import json
+from urllib.parse import urlparse
 
 from django.core.cache import cache
 
@@ -19,7 +23,22 @@ def get_CRIC_panda_queues():
     if not panda_queues_dict:
         panda_queues_dict = {}
         url = settings.CRIC_API_URL
-        http = urllib3.PoolManager()
+        # check http proxy
+        netloc = urlparse(url)
+        proxy = None
+        if 'no_proxy' in os.environ and netloc.hostname in os.environ['no_proxy'].split(','):
+            # no_proxy
+            pass
+        elif netloc.scheme == 'https' and 'https_proxy' in os.environ:
+            # https proxy
+            proxy = os.environ['https']
+        elif netloc.scheme == 'http' and 'http_proxy' in os.environ:
+            # http proxy
+            proxy = os.environ['http']
+        if proxy:
+            http = urllib3.ProxyManager(proxy)
+        else:
+            http = urllib3.PoolManager()
         try:
             r = http.request('GET', url)
             data = json.loads(r.data.decode('utf-8'))
@@ -55,11 +74,14 @@ def get_panda_queues():
         panda_queues_list.extend(SchedconfigJson.objects.values())
         if len(panda_queues_list) > 0:
             for pq in panda_queues_list:
-                try:
-                    panda_queues_dict[pq['pandaqueue']] = json.loads(pq['data'])
-                except:
-                    panda_queues_dict[pq['pandaqueue']] = None
-                    _logger.error("cannot load json from SCHEDCONFIGJSON table for {} PanDA queue".format(pq['pandaqueue']))
+                if isinstance(pq['data'], dict):
+                    panda_queues_dict[pq['pandaqueue']] = pq['data']
+                else:
+                    try:
+                        panda_queues_dict[pq['pandaqueue']] = json.loads(pq['data'])
+                    except:
+                        panda_queues_dict[pq['pandaqueue']] = None
+                        _logger.error("cannot load json from SCHEDCONFIGJSON table for {} PanDA queue".format(pq['pandaqueue']))
 
     return panda_queues_dict
 

core/views.py

@@ -2150,7 +2150,7 @@ def jobInfo(request, pandaid=None, batchid=None, p2=None, p3=None, p4=None):
 
                 if f['datasetid'] in datasets_dict:
                     f['datasetname'] = datasets_dict[f['datasetid']]
-                    if f['scope'] + ":" in f['datasetname']:
+                    if f['scope'] and f['scope'] + ":" in f['datasetname']:
                         f['ruciodatasetname'] = f['datasetname'].split(":")[1]
                     else:
                         f['ruciodatasetname'] = f['datasetname']
@@ -2159,7 +2159,7 @@ def jobInfo(request, pandaid=None, batchid=None, p2=None, p3=None, p4=None):
                             f['ddmsite'] = panda_queues[job['computingsite']]['gocname']
                         else:
                             f['ddmsite'] = computeSvsAtlasS.get(job['computingsite'], "")
-                if 'dst' in f['destinationdblocktoken']:
+                if f['destinationdblocktoken'] and 'dst' in f['destinationdblocktoken']:
                     parced = f['destinationdblocktoken'].split("_")
                     f['ddmsite'] = parced[0][4:]
                     f['dsttoken'] = parced[1]
@@ -2179,7 +2179,7 @@ def jobInfo(request, pandaid=None, batchid=None, p2=None, p3=None, p4=None):
                 if 'creationdate' not in f: f['creationdate'] = f['modificationtime']
                 if 'fileid' not in f: f['fileid'] = f['row_id']
                 if 'datasetname' not in f:
-                    if  f['scope']+":" in f['dataset']:
+                    if f['scope'] and f['scope']+":" in f['dataset']:
                         f['datasetname'] = f['dataset']
                         f['ruciodatasetname'] = f['dataset'].split(":")[1]
                     else: