fix combine contents and contents_ext

main/lib/idds/orm/contents.py

@@ -991,10 +991,14 @@ def combine_contents_ext(contents, contents_ext, with_status_name=False):
     rets = []
     for content in contents:
         content_id = content['content_id']
+        ret = content
         if content_id in contents_ext_map:
+            contents_ext_map[content_id].update(content)
             ret = contents_ext_map[content_id]
         else:
-            ret = {'content_id': content_id}
+            default_params = get_contents_ext_items()
+            default_params.update(content)
+            ret = default_params
         if with_status_name:
             ret['status'] = content['status'].name
         else:

main/lib/idds/tests/panda_client_test.py

@@ -0,0 +1,23 @@
+import pandaclient.idds_api as idds_api
+import idds.common.utils as idds_utils
+
+# setup connection and get jtids
+conn = idds_api.get_api(idds_utils.json_dumps, idds_host=None, compress=True, manager=True)
+reqid = 5460
+ret = conn.get_requests(request_id=int(reqid), with_detail=True)
+jtids = [task["transform_workload_id"] for task in ret[1][1] if task["transform_status"]["attributes"]["_name_"] != "Finished"]
+print(jtids)
+
+# this first jtid works fine
+jtid = jtids[0]
+print(jtid)
+ret = conn.get_contents_output_ext(request_id=reqid, workload_id=jtid)
+print(ret[1][0])
+# print(ret[1][1])
+
+# other jtids do not work
+jtid = jtids[1]
+print(jtid)
+ret = conn.get_contents_output_ext(request_id=reqid, workload_id=jtid)
+print(ret[1][0])
+print(ret[1][1])

main/lib/idds/tests/test_catalog_core.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <[email protected]>, 2023
+
+
+from idds.common.constants import CollectionRelationType
+from idds.common.utils import json_dumps
+from idds.core.catalog import get_contents, get_contents_ext, combine_contents_ext
+
+
+def test(request_id, workload_id, transform_id=None, group_by_jedi_task_id=False):
+    contents = get_contents(request_id=request_id, workload_id=workload_id, transform_id=transform_id,
+                            relation_type=CollectionRelationType.Output)
+    print(len(contents))
+    contents_ext = get_contents_ext(request_id=request_id, workload_id=workload_id, transform_id=transform_id)
+
+    print((len(contents_ext)))
+    ret_contents = combine_contents_ext(contents, contents_ext, with_status_name=True)
+    print(len(ret_contents))
+    rets = {}
+    for content in ret_contents:
+        if group_by_jedi_task_id:
+            jedi_task_id = content.get('jedi_task_id', 'None')
+            if jedi_task_id not in rets:
+                rets[jedi_task_id] = []
+            rets[jedi_task_id].append(content)
+        else:
+            transform_id = content.get('transform_id')
+            if transform_id not in rets:
+                rets[transform_id] = []
+            rets[transform_id].append(content)
+    return rets
+
+
+if __name__ == '__main__':
+    print('164443')
+    rets = test(5460, 164443)
+    # print(json_dumps(rets, sort_keys=True, indent=4))
+    print('164448')
+    rets = test(5460, 164448)
+    # print(str(rets))
+    print(json_dumps(rets, sort_keys=True, indent=4))

main/lib/idds/tests/test_core_debug.py

@@ -0,0 +1,50 @@
+import sys    # noqa F401
+
+from idds.common.constants import ContentStatus
+from idds.common.utils import json_loads
+from idds.orm.contents import update_contents_ext, update_contents    # noqa F401
+
+
+params = [{'content_id': 188308635, 'status': ContentStatus.Available},
+          {'content_id': 188308636, 'status': ContentStatus.Available},
+          {'content_id': 188308637, 'status': ContentStatus.Available},
+          {'content_id': 188308638, 'status': ContentStatus.Available},
+          {'content_id': 188308639, 'status': ContentStatus.Available}]
+params = [{'content_id': 188741016, 'status': ContentStatus.Available},
+          {'content_id': 188741017, 'status': ContentStatus.Available},
+          {'content_id': 188741018, 'status': ContentStatus.Available},
+          {'content_id': 188741019, 'status': ContentStatus.Available},
+          {'content_id': 188741020, 'status': ContentStatus.Available},
+          {'content_id': 188741021, 'status': ContentStatus.Available},
+          {'content_id': 188741022, 'status': ContentStatus.Available},
+          {'content_id': 188741023, 'status': ContentStatus.Available},
+          {'content_id': 188741024, 'status': ContentStatus.Available},
+          {'content_id': 188741025, 'status': ContentStatus.Available},
+          {'content_id': 188741026, 'status': ContentStatus.Available},
+          {'content_id': 188741027, 'status': ContentStatus.Available}]
+# update_contents(params)
+
+# sys.exit(0)
+
+file = '/afs/cern.ch/user/w/wguan/workdisk/iDDS/test/test.log'
+f = open(file, "r")
+c = f.read()
+j = json_loads(c)
+contents_ext = j['update_contents_ext']
+
+new_list = []
+
+for col in contents_ext:
+    new_l = {}
+    for k in col:
+        if col[k] is not None:
+            new_l[k] = col[k]
+        # new_l['job_status'] = 10
+    new_list.append(new_l)
+    # break
+
+print("new_list")
+# print(new_list)
+print("new_list len: %s" % len(new_list))
+update_contents_ext(new_list)
+# update_contents_ext(contents_ext)