iDDS | fix common idds page

core/iDDS/models.py

@@ -13,6 +13,8 @@
 
 class Transforms(models.Model):
     transform_id = models.BigIntegerField(primary_key=True, db_column='transform_id')
+    request_id = models.BigIntegerField(null=True, db_column='request_id')
+    workload_id = models.BigIntegerField(null=True, db_column='workload_id')
     transform_type = models.SmallIntegerField(db_column='transform_type')
     transform_tag = models.CharField(null=True, max_length=20, db_column='transform_tag')
     priority = models.IntegerField(null=True, db_column='priority')
@@ -136,17 +138,3 @@ class Requests(models.Model):
     class Meta:
         db_table = f'"{settings.DB_SCHEMA_IDDS}"."requests"'
         app_label = app_label_idds
-
-
-class Req2transforms(models.Model):
-    request_id_fk = models.ForeignKey(Requests, related_name='request_id_fk', on_delete=models.DO_NOTHING, db_column='request_id')
-    transform_id_fk = models.ForeignKey(Transforms, related_name='transform_id_fk', on_delete=models.DO_NOTHING, db_column='transform_id')
-
-    class Meta:
-        db_table = f'"{settings.DB_SCHEMA_IDDS}"."req2transforms"'
-        app_label = app_label_idds
-        unique_together = (('request_id_fk', 'transform_id_fk'),)
-        managed = False
-
-
-

core/iDDS/rawsqlquery.py

@@ -9,48 +9,29 @@
 subtitleValue = SubstitleValue()
 
 
-def getTransforms(requestid):
-    sqlpar = {"requestid": requestid}
-    sql = """
-    select r.request_id, wt.transform_id
-    from {0}.requests r
-     full outer join (
-        select request_id, workprogress_id from {0}.workprogresses
-     ) wp on (r.request_id=wp.request_id)
-     full outer join {0}.wp2transforms wt on (wp.workprogress_id=wt.workprogress_id)
-    where r.request_id=:requestid
-    """.format(settings.DB_SCHEMA_IDDS)
-    cur = connection.cursor()
-    cur.execute(sql, sqlpar)
-    rows = dictfetchall(cur)
-    cur.close()
-    return rows
-
+def getRequests(query_params, condition='(1=1)'):
 
-def getRequests(query_params):
-    condition = '(1=1)'
     sqlpar = {}
-
     if query_params and len(query_params) > 0:
         query_params = subtitleValue.replaceInverseKeys('requests', query_params)
 
         if 'reqstatus' in query_params:
             sqlpar['rstatus'] = query_params['reqstatus']
-            condition = 'r.status = :rstatus'
+            condition += 'AND r.status = :rstatus'
 
     sql = f"""
     select r.request_id, r.scope, r.name, r.status, tr.transform_id, tr.transform_status, tr.in_status, tr.in_total_files, 
         tr.in_processed_files, tr.out_status, tr.out_total_files, tr.out_processed_files
     from {settings.DB_SCHEMA_IDDS}.requests r
-     full outer join (
+    full outer join (
         select t.request_id, t.transform_id, t.status transform_status, in_coll.status in_status, in_coll.total_files in_total_files,
         in_coll.processed_files in_processed_files, out_coll.status out_status, out_coll.total_files out_total_files,
         out_coll.processed_files out_processed_files
         from {settings.DB_SCHEMA_IDDS}.transforms t
         full outer join (select coll_id , transform_id, status, total_files, processed_files from {settings.DB_SCHEMA_IDDS}.collections where relation_type = 0) in_coll on (t.transform_id = in_coll.transform_id)
         full outer join (select coll_id , transform_id, status, total_files, processed_files from {settings.DB_SCHEMA_IDDS}.collections where relation_type = 1) out_coll on (t.transform_id = out_coll.transform_id)
-     ) tr on (r.request_id=tr.request_id)
-        where {condition}
+    ) tr on (r.request_id=tr.request_id)
+    where {condition}
     """
 
     cur = connection.cursor()
@@ -68,7 +49,7 @@ def prepareSQLQueryParameters(request_params, **kwargs):
     sqlpar, condition = {}, " (1=1)  "
     request_params = {key: value for key, value in request_params.items() if key in ['requestid', 'username', 'status']}
     query_fields_for_subst = ['status']
-    dict_for_subst = {key:request_params.get(key) for key in query_fields_for_subst if key in request_params}
+    dict_for_subst = {key: request_params.get(key) for key in query_fields_for_subst if key in request_params}
     query_params_substituted = subtitleValue.replaceInverseKeys('requests', dict_for_subst)
 
     sqlpar['starttime'] = (datetime.utcnow()-timedelta(hours=24*90)).strftime(settings.DATETIME_FORMAT)
@@ -111,8 +92,9 @@ def getWorkFlowProgressItemized(request_params, **kwargs):
     sql = f"""
     select r.request_id, r.name as r_name, r.status as r_STATUS, r.created_at as r_created_at, c.total_files, 
     c.processed_files, c.processing_files, c.transform_id, t.workload_id, p.status as p_status, r.username 
-    from {settings.DB_SCHEMA_IDDS}.requests r left join {settings.DB_SCHEMA_IDDS}.collections c on r.request_id=c.request_id
-    left join {settings.DB_SCHEMA_IDDS}.transforms t on t.transform_id = c.transform_id 
+    from {settings.DB_SCHEMA_IDDS}.requests r 
+    left join {settings.DB_SCHEMA_IDDS}.collections c on r.request_id=c.request_id
+    left join {settings.DB_SCHEMA_IDDS}.transforms t on t.transform_id=c.transform_id 
     left join {settings.DB_SCHEMA_IDDS}.processings p on p.transform_id=t.transform_id
     where c.relation_type=0 and {condition} order by r.request_id desc
     """

core/iDDS/templates/landing.html

@@ -51,7 +51,7 @@
       </div>
       <div class="card-section">
         <table id="requests_table" class="display data-table" >
-         <thead>
+         <thead><tr>
              <th>Request ID</th>
              <th>Scope</th>
              <th>Name</th>
@@ -63,7 +63,7 @@
              <th>Out status</th>
              <th>Out total files</th>
              <th>Out processed files</th>
-         </thead>
+         </tr></thead>
          <tbody></tbody>
         </table>
       </div>
@@ -90,7 +90,6 @@
             <th>started_at</th>
             <th>finished_at</th>
             <th>expired_at</th>
-            <th>transform_metadata</th>
          </tr></thead>
          <tbody></tbody>
         </table>
@@ -298,7 +297,6 @@
         { data: "started_at", title: "started_at" },
         { data: "finished_at", title: "finished_at" },
         { data: "expired_at", title: "expired_at" },
-        { data: "transform_metadata", title: "transform_metadata" },
     ],
     "processing": true,
     language: {

core/iDDS/templates/workflows.html

@@ -71,7 +71,6 @@
 
 <script type="text/javascript">
 var iDDSrequests=JSON.parse(JSON.stringify({{ iDDSrequests | safe }}));
-const iDDSinstance = '{{ iDDSinstance|safe }}';
 
 function preprocData(dataIn){
    var dataOut = [];

core/iDDS/views.py

@@ -6,14 +6,14 @@
 from django.template.defaulttags import register
 from django.db.models import Q
 from core.oauth.utils import login_customrequired
-from core.views import initRequest
+from core.views import initRequest, setupView
 from core.iDDS.models import Transforms, Collections, Processings, Contents
 from core.iDDS.useconstants import SubstitleValue
-from core.iDDS.rawsqlquery import getRequests, getTransforms
+from core.iDDS.rawsqlquery import getRequests
 from core.iDDS.algorithms import generate_requests_summary, parse_request, getiDDSInfoForTask
 from core.libs.exlib import lower_dicts_in_list
 from core.libs.DateEncoder import DateEncoder
-from django.core.cache import cache
+from core.libs.cache import getCacheEntry, setCacheEntry
 
 _logger = logging.getLogger('bigpandamon')
 
@@ -30,123 +30,141 @@ def to_float(value):
 
 @login_customrequired
 def main(request):
-    initRequest(request)
-    query_params = parse_request(request)
+    valid, response = initRequest(request)
+    if not valid:
+        return response
+
+    data = getCacheEntry(request, "iDDSrequests")
+    if data is not None:
+        data = json.loads(data)
+        response = render_to_response('landing.html', data, content_type='text/html')
+        return response
+
+    query_time = setupView(request, hours=24*7, querytype='idds')
+    condition = "r.created_at > to_date('{1}', '{0}') AND r.created_at <= to_date('{2}', '{0}')".format(
+        'YYYY-MM-DD',
+        query_time['modificationtime__castdate__range'][0][:10],
+        query_time['modificationtime__castdate__range'][1][:10]
+    )
 
-    iDDSrequests = cache.get('iDDSrequests', None)
-    if not iDDSrequests:
-        try:
-            iDDSrequests = getRequests(query_params)
-            cache.set("iDDSrequests", iDDSrequests, 10 * 60)
-        except Exception as e:
-            iDDSrequests = []
-            _logger.exception('Failed to load iDDS requests from DB: \n{}'.format(e))
+    query_params = parse_request(request)
+    try:
+        iDDSrequests = getRequests(query_params, condition=condition)
+    except Exception as e:
+        iDDSrequests = []
+        _logger.exception('Failed to load iDDS requests from DB: \n{}'.format(e))
 
     iDDSrequests = lower_dicts_in_list(iDDSrequests)
     subtitleValue.replace('requests', iDDSrequests)
     requests_summary = generate_requests_summary(iDDSrequests)
 
     data = {
-        'requests_summary':requests_summary,
+        'requests_summary': requests_summary,
         'request': request,
         'viewParams': request.session['viewParams'] if 'viewParams' in request.session else None,
         'iDDSrequests': json.dumps(iDDSrequests, cls=DateEncoder),
     }
+    setCacheEntry(request, "iDDSrequests", json.dumps(data, cls=DateEncoder), 60 * 10)
     response = render_to_response('landing.html', data, content_type='text/html')
     patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
 
     return response
 
 
 def collections(request):
-    initRequest(request)
+    valid, response = initRequest(request)
+    if not valid:
+        return response
     resp = {}
     query = Q()
     if 'transform_id' in request.session['requestParams']:
         query = Q(transform_id=request.session['requestParams']['transform_id'])
     if 'relation_type' in request.session['requestParams']:
         query = Q(relation_type=request.session['requestParams']['relation_type']) & query
 
-    iDDScollections = list(Collections.objects.filter(query)
-                          .values('coll_id',
-                                  'status',
-                                  'total_files',
-                                  'new_files',
-                                  'processed_files',
-                                  'relation_type'
-                                  ))
+    iDDScollections = list(Collections.objects.filter(query).values(
+        'coll_id',
+        'status',
+        'total_files',
+        'new_files',
+        'processed_files',
+        'relation_type'
+        ))
     subtitleValue.replace('collections', iDDScollections)
     return JsonResponse({'data': iDDScollections}, encoder=DateEncoder, safe=False)
 
 
 def iddscontents(request):
-    initRequest(request)
+    valid, response = initRequest(request)
+    if not valid:
+        return response
     query = Q()
     if 'coll_id' in request.session['requestParams']:
         query = Q(coll_id=request.session['requestParams']['coll_id'])
-    iDDSсontents = list(Contents.objects.filter(query)
-                          .values('content_id',
-                                  'scope',
-                                  'name',
-                                  'min_id',
-                                  'max_id',
-                                  'status',
-                                  'storage_id'
-                                  ))
+    iDDSсontents = list(Contents.objects.filter(query).values(
+        'content_id',
+        'scope',
+        'name',
+        'min_id',
+        'max_id',
+        'status',
+        'storage_id'
+        ))
     subtitleValue.replace('сontents', iDDSсontents)
     return JsonResponse({'data': iDDSсontents}, encoder=DateEncoder, safe=False)
 
 
 def processings(request):
-    initRequest(request)
+    valid, response = initRequest(request)
+    if not valid:
+        return response
     query = Q()
     if 'transform_id' in request.session['requestParams']:
         query = Q(transform_id=request.session['requestParams']['transform_id'])
-    iDDSprocessings = list(Processings.objects.filter(query)
-                          .values('processing_id',
-                                  'transform_id',
-                                  'status',
-                                  'created_at',
-                                  'updated_at',
-                                  'finished_at'
-                                  ))
+    iDDSprocessings = list(Processings.objects.filter(query).values(
+        'processing_id',
+        'transform_id',
+        'status',
+        'created_at',
+        'updated_at',
+        'finished_at'
+        ))
     subtitleValue.replace('processings', iDDSprocessings)
     return JsonResponse({'data': iDDSprocessings}, encoder=DateEncoder, safe=False)
 
 
 def transforms(request):
-    initRequest(request)
-    query = Q()
+    valid, response = initRequest(request)
+    if not valid:
+        return response
+    iDDStransforms = []
     if 'requestid' in request.session['requestParams']:
-        values = getTransforms(request.session['requestParams']['requestid'])
-        queries = [Q(transform_id=value['TRANSFORM_ID']) for value in values]
-        query = queries.pop()
-        for item in queries:
-            query |= item
-    iDDStransforms = list(Transforms.objects.filter(query)
-                          .values('transform_id',
-                                  'transform_type',
-                                  'transform_tag',
-                                  'priority',
-                                  'safe2get_output_from_input',
-                                  'status',
-                                  'substatus',
-                                  'locking',
-                                  'retries',
-                                  'created_at',
-                                  'updated_at',
-                                  'started_at',
-                                  'finished_at',
-                                  'expired_at',
-                                  'transform_metadata',
-                                  ))
+        query = {'request_id': request.session['requestParams']['requestid']}
+        iDDStransforms = list(Transforms.objects.filter(**query).values(
+            'transform_id',
+            'transform_type',
+            'transform_tag',
+            'priority',
+            'safe2get_output_from_input',
+            'status',
+            'substatus',
+            'locking',
+            'retries',
+            'created_at',
+            'updated_at',
+            'started_at',
+            'finished_at',
+            'expired_at',
+            ))
     return JsonResponse({'data': iDDStransforms}, encoder=DateEncoder, safe=False)
 
 
 def getiDDSInfoForTaskRequest(request):
-    initRequest(request)
+    valid, response = initRequest(request)
+    if not valid:
+        return response
     transformationWithNested = None
     if 'jeditaskid' in request.session['requestParams']:
         jeditaskid = request.session['requestParams']['jeditaskid']
         transformationWithNested = getiDDSInfoForTask(jeditaskid)
-    return JsonResponse({'data': transformationWithNested}, encoder=DateEncoder, safe=False)
+    return JsonResponse({'data': transformationWithNested}, encoder=DateEncoder, safe=False)

core/iDDS/workflowprogress.py

@@ -95,7 +95,6 @@ def wfprogress(request):
     data = {
         'iDDSrequests': iDDSrequests,
         'iDDSsummary': iDDSsummary,
-        'iDDSinstance': 'default',
         'request': request,
         'viewParams': request.session['viewParams'] if 'viewParams' in request.session else None,
     }

core/views.py

@@ -558,7 +558,7 @@ def setupView(request, opmode='', hours=0, limit=-99, querytype='job', wildCardE
         #    viewParams['selection'] += "hours=%s" % LAST_N_HOURS_MAX
         # else:
         #    viewParams['selection'] += "days=%s" % int(LAST_N_HOURS_MAX/24)
-        if request.session['JOB_LIMIT'] < 100000 and request.session['JOB_LIMIT'] > 0:
+        if querytype == 'job' and request.session['JOB_LIMIT'] < 100000 and request.session['JOB_LIMIT'] > 0:
             request.session['viewParams']['selection'] += " <b>limit=</b>%s" % request.session['JOB_LIMIT']
     else:
         request.session['viewParams']['selection'] = ""