From 205d0d92c765517a89f62d07ea18f7724717160e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 13 Jul 2021 15:03:16 +0200 Subject: [PATCH 001/156] update trigger releasing jobs --- .../idds/agents/transformer/transformer.py | 6 +++ main/lib/idds/core/transforms.py | 37 ++++++++++++- main/lib/idds/tests/core_tests.py | 53 ++++++++++++++++++- workflow/lib/idds/workflow/work.py | 2 +- 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 4c9710a9..acbc5407 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -849,6 +849,12 @@ def process_running_transform_real(self, transform): self.logger.info("syn_work_status: %s, transform status: %s" % (transform['transform_id'], transform['status'])) work.syn_work_status(registered_input_output_maps, all_updates_flushed, output_statistics, to_release_input_contents) + if work.is_terminated(): + self.logger.info("Transform(%s) work is terminated, trigger to release all final status files" % (transform['transform_id'])) + if work.use_dependency_to_release_jobs(): + self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) + to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps) + to_release_input_contents = to_release_input_contents + to_release_input_contents1 to_resume_transform = False reactivated_contents = [] diff --git a/main/lib/idds/core/transforms.py b/main/lib/idds/core/transforms.py index c6844ad6..40a79a30 100644 --- a/main/lib/idds/core/transforms.py +++ b/main/lib/idds/core/transforms.py @@ -428,7 +428,7 @@ def release_inputs(to_release_inputs): return update_contents -def release_inputs_by_collection(to_release_inputs): +def release_inputs_by_collection_old(to_release_inputs): update_contents = [] for coll_id in to_release_inputs: to_release_contents = to_release_inputs[coll_id] @@ -484,6 +484,41 @@ def release_inputs_by_collection(to_release_inputs): return update_contents +def release_inputs_by_collection(to_release_inputs): + update_contents = [] + status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing] + for coll_id in to_release_inputs: + to_release_contents = to_release_inputs[coll_id] + if to_release_contents: + to_release_status = {} + for st in status_to_check: + if st.name not in to_release_status: + to_release_status[st.name] = [] + + for to_release_content in to_release_contents: + for st in status_to_check: + if (to_release_content['status'] in [st] or to_release_content['substatus'] in [st]): + to_release_status[st.name].append(to_release_content['name']) + + print("to_release_status: %s" % str(to_release_status)) + + contents = orm_contents.get_input_contents(request_id=to_release_contents[0]['request_id'], + coll_id=to_release_contents[0]['coll_id'], + name=None) + # print("contents: %s" % str(contents)) + + for content in contents: + if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503 + for st in status_to_check: + if (content['status'] not in [st] and content['name'] in to_release_status[st.name]): + update_content = {'content_id': content['content_id'], + 'substatus': st, + 'status': st} + update_contents.append(update_content) + + return update_contents + + def get_work_name_to_coll_map(request_id): tfs = orm_transforms.get_transforms(request_id=request_id) colls = orm_collections.get_collections(request_id=request_id) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 741190aa..0a9e5658 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -1,13 +1,64 @@ import sys from idds.common.utils import json_dumps # noqa F401 -from idds.common.constants import ContentStatus # noqa F401 +from idds.common.constants import ContentStatus, ContentType, ContentRelationType # noqa F401 from idds.core.requests import get_requests # noqa F401 from idds.core.messages import retrieve_messages # noqa F401 from idds.core.transforms import get_transforms # noqa F401 from idds.core.workprogress import get_workprogresses # noqa F401 from idds.core.processings import get_processings # noqa F401 from idds.core import transforms as core_transforms # noqa F401 +from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401 + + +def release_inputs_test(): + to_release_inputs = {3498: [{'map_id': 1, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', + 'substatus': ContentStatus.Available, 'path': None, + 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+3_isr_3020111900038_94+qgraphNodeId:3+qgraphId:1626129062.5744567-119392', + 'content_id': 2248918, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, + 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', + 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, + 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411522}, 'transform_id': 1749, 'storage_id': None}, + {'map_id': 2, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', + 'substatus': ContentStatus.Available, 'path': None, + 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+2_isr_3020111900032_94+qgraphNodeId:2+qgraphId:1626129062.5744567-119392', + 'content_id': 2248919, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, + 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', + 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, + 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411523}, 'transform_id': 1749, 'storage_id': None}, + {'map_id': 3, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', + 'substatus': ContentStatus.Available, 'path': None, + 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+4_isr_3020111900040_94+qgraphNodeId:4+qgraphId:1626129062.5744567-119392', + 'content_id': 2248920, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, + 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', + 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, + 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411524}, 'transform_id': 1749, 'storage_id': None}, + {'map_id': 4, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', + 'substatus': ContentStatus.Available, 'path': None, + 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+1_isr_3020111900036_94+qgraphNodeId:1+qgraphId:1626129062.5744567-119392', + 'content_id': 2248921, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, + 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', + 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, + 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411525}, 'transform_id': 1749, 'storage_id': None}, + {'map_id': 5, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', + 'substatus': ContentStatus.Available, 'path': None, + 'name': 'u_jchiang_dark_12781_panda_20210712T222923Z.qgraph+0_isr_3020111900034_94+qgraphNodeId:0+qgraphId:1626129062.5744567-119392', + 'content_id': 2248922, 'min_id': 0, 'bytes': 1, 'coll_id': 3498, 'max_id': 1, 'md5': None, + 'request_id': 93, 'content_type': ContentType.File, 'adler32': '12345678', + 'workload_id': 1626129080, 'content_relation_type': ContentRelationType.Output, + 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411526}, 'transform_id': 1749, 'storage_id': None} + ]} + + update_contents = release_inputs_by_collection(to_release_inputs) + print(update_contents) + + update_contents = release_inputs_by_collection_old(to_release_inputs) + print(update_contents) + + +release_inputs_test() + +sys.exit(0) def show_works(req): diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index f7de08c8..bc4eb066 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -936,7 +936,7 @@ def is_terminated(self): """ *** Function called by Transformer agent. """ - if (self.status in [WorkStatus.Finished, WorkStatus.SubFinished, WorkStatus.Failed, WorkStatus.Cancelled, WorkStatus.Suspended] + if (self.status in [WorkStatus.Finished, WorkStatus.SubFinished, WorkStatus.Failed, WorkStatus.Cancelled, WorkStatus.Suspended, WorkStatus.Expired] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]): # noqa W503 return True return False From 539b275e8f56f2fc4674e7ca62111307a49110a3 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 13 Jul 2021 16:48:25 +0200 Subject: [PATCH 002/156] improve trigger release inputs --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- .../idds/agents/transformer/transformer.py | 6 ++--- main/lib/idds/core/transforms.py | 23 ++++++++----------- main/lib/idds/tests/core_tests.py | 2 +- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 ++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 16 files changed, 32 insertions(+), 35 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 50053b7e..57dccb12 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 242b7955..4a8ea9fe 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.5.0 - - idds-workflow==0.5.0 \ No newline at end of file + - idds-common==0.5.1 + - idds-workflow==0.5.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 50053b7e..57dccb12 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 2d4c5ff5..1b439490 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.5.0 - - idds-workflow==0.5.0 \ No newline at end of file + - idds-common==0.5.1 + - idds-workflow==0.5.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 50053b7e..57dccb12 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 09eb8cd6..1f35ac7f 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index bbde400e..ad9f53ac 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.5.0 - - idds-workflow==0.5.0 \ No newline at end of file + - idds-common==0.5.1 + - idds-workflow==0.5.1 \ No newline at end of file diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index acbc5407..80c6fef3 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -276,7 +276,7 @@ def trigger_release_inputs_old(self, updated_output_contents, work, input_output self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) return updated_contents - def trigger_release_inputs(self, updated_output_contents, work, input_output_maps): + def trigger_release_inputs(self, updated_output_contents, work, input_output_maps, final=False): to_release_inputs = {} for map_id in input_output_maps: outputs = input_output_maps[map_id]['outputs'] if 'outputs' in input_output_maps[map_id] else [] @@ -288,7 +288,7 @@ def trigger_release_inputs(self, updated_output_contents, work, input_output_map to_release_inputs[content['coll_id']].append(content) # updated_contents = core_transforms.release_inputs(to_release_inputs) - updated_contents = core_transforms.release_inputs_by_collection(to_release_inputs) + updated_contents = core_transforms.release_inputs_by_collection(to_release_inputs, final=final) self.logger.debug("trigger_release_inputs, to_release_inputs: %s" % str(to_release_inputs)) self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) return updated_contents @@ -853,7 +853,7 @@ def process_running_transform_real(self, transform): self.logger.info("Transform(%s) work is terminated, trigger to release all final status files" % (transform['transform_id'])) if work.use_dependency_to_release_jobs(): self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) - to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps) + to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps, final=True) to_release_input_contents = to_release_input_contents + to_release_input_contents1 to_resume_transform = False diff --git a/main/lib/idds/core/transforms.py b/main/lib/idds/core/transforms.py index 40a79a30..ff9e5769 100644 --- a/main/lib/idds/core/transforms.py +++ b/main/lib/idds/core/transforms.py @@ -484,23 +484,20 @@ def release_inputs_by_collection_old(to_release_inputs): return update_contents -def release_inputs_by_collection(to_release_inputs): +def release_inputs_by_collection(to_release_inputs, final=False): update_contents = [] status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing] for coll_id in to_release_inputs: to_release_contents = to_release_inputs[coll_id] if to_release_contents: to_release_status = {} - for st in status_to_check: - if st.name not in to_release_status: - to_release_status[st.name] = [] - for to_release_content in to_release_contents: - for st in status_to_check: - if (to_release_content['status'] in [st] or to_release_content['substatus'] in [st]): - to_release_status[st.name].append(to_release_content['name']) + if (to_release_content['status'] in status_to_check): + to_release_status[to_release_content['name']] = to_release_content['status'] + elif (to_release_content['substatus'] in status_to_check): + to_release_status[to_release_content['name']] = to_release_content['substatus'] - print("to_release_status: %s" % str(to_release_status)) + # print("to_release_status: %s" % str(to_release_status)) contents = orm_contents.get_input_contents(request_id=to_release_contents[0]['request_id'], coll_id=to_release_contents[0]['coll_id'], @@ -509,11 +506,11 @@ def release_inputs_by_collection(to_release_inputs): for content in contents: if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503 - for st in status_to_check: - if (content['status'] not in [st] and content['name'] in to_release_status[st.name]): + if content['name'] in to_release_status: + if final or (content['status'] != to_release_status[content['name']]): update_content = {'content_id': content['content_id'], - 'substatus': st, - 'status': st} + 'substatus': to_release_status[content['name']], + 'status': to_release_status[content['name']]} update_contents.append(update_content) return update_contents diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 0a9e5658..e5bfe80c 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -52,7 +52,7 @@ def release_inputs_test(): update_contents = release_inputs_by_collection(to_release_inputs) print(update_contents) - update_contents = release_inputs_by_collection_old(to_release_inputs) + update_contents = release_inputs_by_collection(to_release_inputs, final=True) print(update_contents) diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 50053b7e..57dccb12 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 16991170..f0824902 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.5.0 - - idds-workflow==0.5.0 - - idds-client==0.5.0 \ No newline at end of file + - idds-common==0.5.1 + - idds-workflow==0.5.1 + - idds-client==0.5.1 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 50053b7e..57dccb12 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/website/version.py b/website/version.py index 50053b7e..57dccb12 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 50053b7e..57dccb12 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.0" +release_version = "0.5.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 8fd3f4a1..f0c11199 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.5.0 \ No newline at end of file + - idds-common==0.5.1 \ No newline at end of file From da6e5181fb5e7881e6a7a9973e24405240161e7e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 13 Jul 2021 16:49:02 +0200 Subject: [PATCH 003/156] patch version 0.5.2 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 57dccb12..a01ed7a7 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 4a8ea9fe..195e4f33 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.5.1 - - idds-workflow==0.5.1 \ No newline at end of file + - idds-common==0.5.2 + - idds-workflow==0.5.2 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 57dccb12..a01ed7a7 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 1b439490..e864b3c4 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.5.1 - - idds-workflow==0.5.1 \ No newline at end of file + - idds-common==0.5.2 + - idds-workflow==0.5.2 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 57dccb12..a01ed7a7 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 1f35ac7f..2a9d02da 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index ad9f53ac..d9e5fcd3 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.5.1 - - idds-workflow==0.5.1 \ No newline at end of file + - idds-common==0.5.2 + - idds-workflow==0.5.2 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 57dccb12..a01ed7a7 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index f0824902..144485c3 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.5.1 - - idds-workflow==0.5.1 - - idds-client==0.5.1 \ No newline at end of file + - idds-common==0.5.2 + - idds-workflow==0.5.2 + - idds-client==0.5.2 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 57dccb12..a01ed7a7 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/website/version.py b/website/version.py index 57dccb12..a01ed7a7 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 57dccb12..a01ed7a7 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.1" +release_version = "0.5.2" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index f0c11199..097d32ca 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.5.1 \ No newline at end of file + - idds-common==0.5.2 \ No newline at end of file From af8e1a13b7349e417a06e0217d46bccb30cccdbf Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 13 Jul 2021 17:01:17 +0200 Subject: [PATCH 004/156] update tests --- main/lib/idds/tests/panda_test.py | 1 + main/lib/idds/tests/test_migrate_requests.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/main/lib/idds/tests/panda_test.py b/main/lib/idds/tests/panda_test.py index 93026f28..4b74a0fd 100644 --- a/main/lib/idds/tests/panda_test.py +++ b/main/lib/idds/tests/panda_test.py @@ -66,6 +66,7 @@ # task_ids = [i for i in range(1840, 1850)] + [i for i in range(1990, 2000)] # task_ids = [2549, 2560] # task_ids = [i for i in range(3692, 3723)] +# task_ids = [3834, 3835, 3836] task_ids = [] for task_id in task_ids: print("Killing %s" % task_id) diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index 6e24e92e..b161c92f 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -32,11 +32,11 @@ def migrate(): cm1 = ClientManager(host=doma_host) # reqs = cm1.get_requests(request_id=290) old_request_id = 72533 - for old_request_id in [27]: + for old_request_id in [93]: # for old_request_id in [60]: # noqa E115 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=dev_host) + cm2 = ClientManager(host=doma_host) for req in reqs: req = convert_old_req_2_workflow_req(req) workflow = req['request_metadata']['workflow'] From 43d9e40f173e4ecc58a14f1c2b1938ff068be592 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 14 Jul 2021 02:03:01 +0200 Subject: [PATCH 005/156] fix reactive contents --- doma/lib/idds/doma/workflow/domapandawork.py | 8 +++++++- main/lib/idds/agents/transformer/transformer.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index f31a48f6..78f6b8e5 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -521,11 +521,17 @@ def reactive_contents(self, input_output_maps): break if not all_outputs_available: - for content in inputs + outputs + inputs_dependency: + for content in inputs + outputs: update_content = {'content_id': content['content_id'], 'status': ContentStatus.New, 'substatus': ContentStatus.New} updated_contents.append(update_content) + for content in inputs_dependency: + if content['status'] not in [ContentStatus.Available]: + update_content = {'content_id': content['content_id'], + 'status': ContentStatus.New, + 'substatus': ContentStatus.New} + updated_contents.append(update_content) return updated_contents def sort_panda_jobids(self, input_output_maps): diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 80c6fef3..a622b0bc 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -692,11 +692,17 @@ def reactive_contents(self, input_output_maps): break if not all_outputs_available: - for content in inputs + outputs + inputs_dependency: + for content in inputs + outputs: update_content = {'content_id': content['content_id'], 'status': ContentStatus.New, 'substatus': ContentStatus.New} updated_contents.append(update_content) + for content in inputs_dependency: + if content['status'] not in [ContentStatus.Available]: + update_content = {'content_id': content['content_id'], + 'status': ContentStatus.New, + 'substatus': ContentStatus.New} + updated_contents.append(update_content) return updated_contents def process_running_transform_real(self, transform): From 68fc5b4fed658e1e5f9a1beba252a5e2c428f600 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 14 Jul 2021 13:30:36 +0200 Subject: [PATCH 006/156] fix max attempt for panda --- doma/lib/idds/doma/workflow/domapandawork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index 78f6b8e5..8737f606 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -586,7 +586,7 @@ def get_content_status_from_panda_status(self, job_info): elif jobstatus in ['failed', 'closed', 'cancelled', 'lost', 'broken', 'missing']: attempt_nr = int(job_info.attemptNr) if job_info.attemptNr else 0 max_attempt = int(job_info.maxAttempt) if job_info.maxAttempt else 0 - if attempt_nr >= max_attempt: + if (attempt_nr >= max_attempt) and (attempt_nr >= self.maxAttempt): return ContentStatus.FinalFailed else: return ContentStatus.Failed From 322c6d2fafd36f5783a7f9b678467974066cfc8b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 14 Jul 2021 14:10:14 +0200 Subject: [PATCH 007/156] duplicate toresume messages --- main/lib/idds/agents/clerk/clerk.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index a68178fa..80c811fc 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -424,14 +424,13 @@ def process_operating_request_real(self, req): new_messages = [] tfs = core_transforms.get_transforms(request_id=req['request_id']) for tf in tfs: - # if tf['status'] not in [RequestStatus.Finished, RequestStatus.SubFinished, - # RequestStatus.Failed, RequestStatus.Cancelling, - # RequestStatus.Cancelled, RequestStatus.Suspending, - # RequestStatus.Suspended]: try: # core_transforms.update_transform(transform_id=tf['transform_id'], parameters={'substatus': tf_status}) msg = self.get_message_for_update_transform(tf, tf_status) new_messages.append(msg) + if tf_status in [TransformStatus.ToResume]: + # duplicate the messages for ToResume + new_messages.append(msg) except Exception as ex: self.logger.warn("Failed to add messages for tranform %s, record it for later update: %s" % (tf['transform_id'], str(ex))) From 3b677ecae7e3b5b2e8219ef581380c911ac453d1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 14 Jul 2021 14:14:58 +0200 Subject: [PATCH 008/156] patch version 0.5.3 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index a01ed7a7..c7577f5b 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 195e4f33..64cda279 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.5.2 - - idds-workflow==0.5.2 \ No newline at end of file + - idds-common==0.5.3 + - idds-workflow==0.5.3 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index a01ed7a7..c7577f5b 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index e864b3c4..25426797 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.5.2 - - idds-workflow==0.5.2 \ No newline at end of file + - idds-common==0.5.3 + - idds-workflow==0.5.3 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index a01ed7a7..c7577f5b 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 2a9d02da..928e9158 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index d9e5fcd3..66fcf40f 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.5.2 - - idds-workflow==0.5.2 \ No newline at end of file + - idds-common==0.5.3 + - idds-workflow==0.5.3 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index a01ed7a7..c7577f5b 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 144485c3..ae4c4d3a 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.5.2 - - idds-workflow==0.5.2 - - idds-client==0.5.2 \ No newline at end of file + - idds-common==0.5.3 + - idds-workflow==0.5.3 + - idds-client==0.5.3 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index a01ed7a7..c7577f5b 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/website/version.py b/website/version.py index a01ed7a7..c7577f5b 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index a01ed7a7..c7577f5b 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.2" +release_version = "0.5.3" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 097d32ca..36aad122 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.5.2 \ No newline at end of file + - idds-common==0.5.3 \ No newline at end of file From 52ab8db62886d14208ad913011b51ce188f72b5c Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 15 Jul 2021 14:19:06 +0200 Subject: [PATCH 009/156] update trigger --- main/lib/idds/tests/trigger_release.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 62bdbb88..168fc24d 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -11,17 +11,19 @@ from idds.orm.contents import get_input_contents # noqa F401 -contents = get_contents(request_id=10, status=ContentStatus.Available) -ret_contents = [] +contents = get_contents(request_id=107, status=ContentStatus.Available) +ret_contents = {} for content in contents: if content['content_relation_type'] == ContentRelationType.Output: # InputDependency - ret_contents.append(content) + if content['coll_id'] not in ret_contents: + ret_contents[content['coll_id']] = [] + ret_contents[content['coll_id']].append(content) for ret_content in ret_contents: print(ret_content) break -updated_contents = core_transforms.release_inputs(ret_contents) +updated_contents = core_transforms.release_inputs_by_collection(ret_contents) for update_content in updated_contents: print(update_content) break From e2dcf8f8eb22b0662b7259628b03bb8c02c063d5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 15 Jul 2021 22:43:13 +0200 Subject: [PATCH 010/156] fix work status --- .../lib/idds/atlas/workflow/atlasactuatorwork.py | 2 ++ atlas/lib/idds/atlas/workflow/atlashpowork.py | 2 ++ atlas/lib/idds/atlas/workflow/atlaspandawork.py | 2 ++ atlas/lib/idds/atlas/workflow/atlasstageinwork.py | 15 +++++++++++++-- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py b/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py index e571df43..e80a125a 100644 --- a/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py @@ -304,6 +304,8 @@ def syn_work_status(self, registered_input_output_maps): self.status = WorkStatus.Failed elif self.is_processings_subfinished(): self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming ####### functions for carrier ######## # noqa E266 ###################################### # noqa E266 diff --git a/atlas/lib/idds/atlas/workflow/atlashpowork.py b/atlas/lib/idds/atlas/workflow/atlashpowork.py index 0160bfd8..f27bb3ae 100644 --- a/atlas/lib/idds/atlas/workflow/atlashpowork.py +++ b/atlas/lib/idds/atlas/workflow/atlashpowork.py @@ -424,6 +424,8 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True self.status = WorkStatus.Failed else: self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming ####### functions for carrier ######## # noqa E266 ###################################### # noqa E266 diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index e369600b..40d381f3 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -511,3 +511,5 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True self.status = WorkStatus.Failed elif self.is_processings_subfinished(): self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming diff --git a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py index 83bd039f..5af9139b 100644 --- a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py @@ -120,7 +120,8 @@ def poll_external_collection(self, coll): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return coll def get_input_collections(self): # return [self.primary_input_collection] + self.other_input_collections @@ -302,7 +303,15 @@ def poll_rule(self, processing): raise exceptions.ProcessNotFound(msg) def poll_processing(self, processing): - return self.poll_rule(processing) + try: + return self.poll_rule(processing) + except exceptions.ProcessNotFound as ex: + raise ex + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + + return processing, 'notOk', {} def poll_processing_updates(self, processing, input_output_maps): processing, rule_state, rep_status = self.poll_processing(processing) @@ -410,4 +419,6 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True self.status = WorkStatus.Failed else: self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming self.logger.debug("syn_work_status(%s): work.status: %s" % (str(self.get_processing_ids()), str(self.status))) From 78beed77d36347cc61599d0a1df87efd37fa8498 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 15 Jul 2021 22:43:55 +0200 Subject: [PATCH 011/156] add tests --- .../tests/resume_subfinished_data_carousel.sh | 8 +++ main/lib/idds/tests/run_sql.py | 55 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 main/lib/idds/tests/resume_subfinished_data_carousel.sh create mode 100644 main/lib/idds/tests/run_sql.py diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh new file mode 100644 index 00000000..2eeca111 --- /dev/null +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +request_ids=(75039 75065 75569 75683 76087 76181 76211 76455 76727 77281 77283 77285 77287 77385 77419 77643 78177 78241 78497 78517 78533 78535 78743 78745 78787 78895 78947 79037 79045 79057 79065 79147 79155 79161 79179 79191 79207 79263 79297 79317 79337 79347 79395 79401 79463 79465 79525 79563 79737 79743 79755 79759 79967 79975 79977 79987 80001 80083 80087 80099 80107 80109 80123 80131 80211 80215 80217 80257 80291 80339 80341 80433 80565 80635 80705 80747 80883 80887 80923 81015 81017 81033 81035 81065 81231 81237 81303 81307 81309 81379 81547 82169) + +for request_id in "${request_ids[@]}"; do + echo idds resume_requests --request_id=${request_id} + idds resume_requests --request_id=${request_id} +done diff --git a/main/lib/idds/tests/run_sql.py b/main/lib/idds/tests/run_sql.py new file mode 100644 index 00000000..813a5437 --- /dev/null +++ b/main/lib/idds/tests/run_sql.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2019 + + +""" +performance test to insert contents. +""" +import cx_Oracle + + +from idds.common.config import config_get +# from idds.core.contents import add_content + + +def get_subfinished_requests(db_pool): + connection = db_pool.acquire() + + req_ids = [] + sql = """select request_id from atlas_IDDS.requests where status=4 and scope!='hpo'""" + cursor = connection.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + for row in rows: + # print(row) + req_ids.append(row[0]) + cursor.close() + + connection.commit() + db_pool.release(connection) + print(req_ids) + + +def get_session_pool(): + sql_connection = config_get('database', 'default') + sql_connection = sql_connection.replace("oracle://", "") + user_pass, tns = sql_connection.split('@') + user, passwd = user_pass.split(':') + db_pool = cx_Oracle.SessionPool(user, passwd, tns, min=12, max=20, increment=1) + return db_pool + + +def test(): + pool = get_session_pool() + get_subfinished_requests(pool) + + +if __name__ == '__main__': + test() From 9d12be9909a1e084891086d031e34849efd37fbb Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 16 Jul 2021 15:09:23 +0200 Subject: [PATCH 012/156] fix that different session returns different result --- main/lib/idds/orm/base/session.py | 4 +++- main/lib/idds/tests/core_tests.py | 23 ++++++++++++++++++++++- monitor/conf.js | 12 ++++++------ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/main/lib/idds/orm/base/session.py b/main/lib/idds/orm/base/session.py index cecf0d08..c4013884 100644 --- a/main/lib/idds/orm/base/session.py +++ b/main/lib/idds/orm/base/session.py @@ -252,7 +252,9 @@ def new_funct(*args, **kwargs): session = get_session() try: kwargs['session'] = session - return function(*args, **kwargs) + result = function(*args, **kwargs) + session.close() + return result except TimeoutError as error: session.rollback() # pylint: disable=maybe-no-member raise DatabaseException(str(error)) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index e5bfe80c..599b3355 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -1,13 +1,15 @@ import sys +import datetime from idds.common.utils import json_dumps # noqa F401 -from idds.common.constants import ContentStatus, ContentType, ContentRelationType # noqa F401 +from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking # noqa F401 from idds.core.requests import get_requests # noqa F401 from idds.core.messages import retrieve_messages # noqa F401 from idds.core.transforms import get_transforms # noqa F401 from idds.core.workprogress import get_workprogresses # noqa F401 from idds.core.processings import get_processings # noqa F401 from idds.core import transforms as core_transforms # noqa F401 +from idds.orm.contents import get_input_contents from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401 @@ -49,6 +51,25 @@ def release_inputs_test(): 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1411526}, 'transform_id': 1749, 'storage_id': None} ]} + to_release_inputs = {4042: [{'map_id': 1, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+13_isr_257768_161+1626299263.3909254-24148+13', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254913, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412272}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 + {'map_id': 2, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+2_isr_212071_54+1626299263.3909254-24148+2', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254914, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412273}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 + {'map_id': 3, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+10_isr_456716_99+1626299263.3909254-24148+10', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254915, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412274}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 + {'map_id': 4, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+34_isr_407919_130+1626299263.3909254-24148+34', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254916, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412275}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 + {'map_id': 5, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+23_isr_254379_48+1626299263.3909254-24148+23', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254917, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412276}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 + {'map_id': 6, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+11_isr_37657_141+1626299263.3909254-24148+11', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254918, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412277}, 'transform_id': 2021, 'storage_id': None}, # noqa E501 + {'map_id': 7, 'status': ContentStatus.Available, 'retries': 0, 'scope': 'pseudo_dataset', 'substatus': ContentStatus.Available, 'path': None, 'name': 'u_huanlin_panda_test_ci_imsim_w26_20210714T214732Z.qgraph+31_isr_226983_36+1626299263.3909254-24148+31', 'locking': ContentLocking.Idle, 'created_at': datetime.datetime(2021, 7, 14, 21, 48, 10), 'content_id': 2254919, 'min_id': 0, 'bytes': 1, 'updated_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'coll_id': 4042, 'max_id': 1, 'md5': None, 'accessed_at': datetime.datetime(2021, 7, 14, 22, 8, 30), 'request_id': 107, 'content_type': ContentType.File, 'adler32': '12345678', 'expired_at': datetime.datetime(2021, 8, 13, 21, 48, 10), 'workload_id': 1626299273, 'content_relation_type': ContentRelationType.Output, 'processing_id': None, 'content_metadata': {'events': 1, 'panda_id': 1412278}, 'transform_id': 2021, 'storage_id': None}]} # noqa E501 + + for coll_id in to_release_inputs: + contents = get_input_contents(request_id=to_release_inputs[coll_id][0]['request_id'], + coll_id=coll_id, + name=None) + print(len(contents)) + in_dep_contents = [] + for content in contents: + if (content['content_relation_type'] == ContentRelationType.InputDependency): + in_dep_contents.append(content) + print(len(in_dep_contents)) + update_contents = release_inputs_by_collection(to_release_inputs) print(update_contents) diff --git a/monitor/conf.js b/monitor/conf.js index c0f8f13b..66db231f 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus722.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus722.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus722.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus722.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus722.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus722.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus728.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus728.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus728.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/false/true" } From cf01d77b3b9ca53a96e16bc854fbed8e120418da Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 16 Jul 2021 15:15:52 +0200 Subject: [PATCH 013/156] new patch 0.5.4 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/conf.js | 12 ++++++------ monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index c7577f5b..b025b4fa 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 64cda279..008df559 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.5.3 - - idds-workflow==0.5.3 \ No newline at end of file + - idds-common==0.5.4 + - idds-workflow==0.5.4 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index c7577f5b..b025b4fa 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 25426797..fc9d2746 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.5.3 - - idds-workflow==0.5.3 \ No newline at end of file + - idds-common==0.5.4 + - idds-workflow==0.5.4 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index c7577f5b..b025b4fa 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 928e9158..9cccd94e 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 66fcf40f..7ec1bdf9 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.5.3 - - idds-workflow==0.5.3 \ No newline at end of file + - idds-common==0.5.4 + - idds-workflow==0.5.4 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index c7577f5b..b025b4fa 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index ae4c4d3a..07aa3f84 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.5.3 - - idds-workflow==0.5.3 - - idds-client==0.5.3 \ No newline at end of file + - idds-common==0.5.4 + - idds-workflow==0.5.4 + - idds-client==0.5.4 \ No newline at end of file diff --git a/monitor/conf.js b/monitor/conf.js index 66db231f..e3a91e83 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus728.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus728.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus728.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus750.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus750.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus750.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus750.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus750.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus750.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/version.py b/monitor/version.py index c7577f5b..b025b4fa 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/website/version.py b/website/version.py index c7577f5b..b025b4fa 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index c7577f5b..b025b4fa 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.3" +release_version = "0.5.4" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 36aad122..3270efa5 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.5.3 \ No newline at end of file + - idds-common==0.5.4 \ No newline at end of file From c2a6d5affb22255328778184a9f6463592e656aa Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:15:30 +0200 Subject: [PATCH 014/156] add steering container for toymc --- .../container/docker/{ => hpo}/Dockerfile | 0 .../docker/{ => hpo}/Dockerfile.back | 0 .../tools/container/docker/{ => hpo}/commands | 0 .../{ => hpo}/hyperparameteropt_nevergrad.py | 0 .../docker/{ => hpo}/idds_input.json | 0 main/tools/container/docker/toymc/Dockerfile | 8 +++ .../container/docker/toymc/Dockerfile.back | 35 +++++++++++++ main/tools/container/docker/toymc/commands | 6 +++ .../docker/toymc/hyperparameteropt_toymc.py | 50 +++++++++++++++++++ .../container/docker/toymc/idds_input.json | 1 + 10 files changed, 100 insertions(+) rename main/tools/container/docker/{ => hpo}/Dockerfile (100%) rename main/tools/container/docker/{ => hpo}/Dockerfile.back (100%) rename main/tools/container/docker/{ => hpo}/commands (100%) rename main/tools/container/docker/{ => hpo}/hyperparameteropt_nevergrad.py (100%) rename main/tools/container/docker/{ => hpo}/idds_input.json (100%) create mode 100644 main/tools/container/docker/toymc/Dockerfile create mode 100644 main/tools/container/docker/toymc/Dockerfile.back create mode 100644 main/tools/container/docker/toymc/commands create mode 100644 main/tools/container/docker/toymc/hyperparameteropt_toymc.py create mode 100644 main/tools/container/docker/toymc/idds_input.json diff --git a/main/tools/container/docker/Dockerfile b/main/tools/container/docker/hpo/Dockerfile similarity index 100% rename from main/tools/container/docker/Dockerfile rename to main/tools/container/docker/hpo/Dockerfile diff --git a/main/tools/container/docker/Dockerfile.back b/main/tools/container/docker/hpo/Dockerfile.back similarity index 100% rename from main/tools/container/docker/Dockerfile.back rename to main/tools/container/docker/hpo/Dockerfile.back diff --git a/main/tools/container/docker/commands b/main/tools/container/docker/hpo/commands similarity index 100% rename from main/tools/container/docker/commands rename to main/tools/container/docker/hpo/commands diff --git a/main/tools/container/docker/hyperparameteropt_nevergrad.py b/main/tools/container/docker/hpo/hyperparameteropt_nevergrad.py similarity index 100% rename from main/tools/container/docker/hyperparameteropt_nevergrad.py rename to main/tools/container/docker/hpo/hyperparameteropt_nevergrad.py diff --git a/main/tools/container/docker/idds_input.json b/main/tools/container/docker/hpo/idds_input.json similarity index 100% rename from main/tools/container/docker/idds_input.json rename to main/tools/container/docker/hpo/idds_input.json diff --git a/main/tools/container/docker/toymc/Dockerfile b/main/tools/container/docker/toymc/Dockerfile new file mode 100644 index 00000000..0fbefcd3 --- /dev/null +++ b/main/tools/container/docker/toymc/Dockerfile @@ -0,0 +1,8 @@ +FROM recognai/python-37-centos + +LABEL project="iDDS_HPO_ToyMC(wen.guan@cern.ch)" + +COPY ./hyperparameteropt_toymc.py /opt/ +RUN pip install --upgrade pip +# RUN pip install nevergrad + diff --git a/main/tools/container/docker/toymc/Dockerfile.back b/main/tools/container/docker/toymc/Dockerfile.back new file mode 100644 index 00000000..aff909d7 --- /dev/null +++ b/main/tools/container/docker/toymc/Dockerfile.back @@ -0,0 +1,35 @@ +FROM centos:7 + +LABEL project="iDDS_HPO_Nevergrad(wen.guan@cern.ch)" + +ARG PROJECT=idds + +RUN yum update -q -y \ + && yum install -q -y wget make git gcc openssl-devel bzip2-devel libffi-devel \ + && cd /usr/src \ + && wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ + && tar xzf Python-3.7.4.tgz \ + && cd Python-3.7.4 \ + && ./configure --enable-optimizations \ + && make altinstall \ + && rm -rf /usr/src/Python-3.7.4.tgz \ + && yum clean all \ + # set python3.7 as default + && alternatives --install /usr/bin/python python /usr/bin/python2 50 \ + && alternatives --install /usr/bin/python python /usr/local/bin/python3.7 70 \ + && alternatives --set python /usr/local/bin/python3.7 \ + # symlink pip + && ln -s /usr/local/bin/pip3.7 /usr/bin/pip \ + && pip install --no-cache-dir --upgrade pip pipenv setuptools wheel \ + && ln -s /usr/local/bin/pipenv /usr/bin/pipenv \ + # set the locale + && localedef --quiet -c -i en_US -f UTF-8 en_US.UTF-8 + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +COPY ../../../../atlas/lib/idds/atlas/processing/hyperparameteropt_nevergrad.py /$PROJECT +RUN pip install --upgrade pip +RUN pip install nevergrad + diff --git a/main/tools/container/docker/toymc/commands b/main/tools/container/docker/toymc/commands new file mode 100644 index 00000000..4bc02f65 --- /dev/null +++ b/main/tools/container/docker/toymc/commands @@ -0,0 +1,6 @@ +docker build --tag idds_hpo_nevergrad . +docker run -v /tmp/wguan:/data idds_hpo_nevergrad python /opt/hyperparameteropt_nevergrad.py --max_points=20 --num_points=10 --input=/data/idds_input.json --output=/data/output.json +docker login --username=wguanicedew +docker images +docker tag idds_hpo_nevergrad wguanicedew/idds_hpo_nevergrad +docker push wguanicedew/idds_hpo_nevergrad diff --git a/main/tools/container/docker/toymc/hyperparameteropt_toymc.py b/main/tools/container/docker/toymc/hyperparameteropt_toymc.py new file mode 100644 index 00000000..6e6ae39f --- /dev/null +++ b/main/tools/container/docker/toymc/hyperparameteropt_toymc.py @@ -0,0 +1,50 @@ +import argparse +import json +# import nevergrad as ng + + +parser = argparse.ArgumentParser() +parser.add_argument('--max_points', action='store', type=int, required=True, help='max number of points to be generated') +parser.add_argument('--num_points', action='store', type=int, required=True, help='number of points to be generated') +parser.add_argument('--input', action='store', required=True, help='input json file which includes all pre-generated points') +parser.add_argument('--output', action='store', required=True, help='output json file where outputs will be wrote') + +args = parser.parse_args() + + +def get_input_points(input): + points = [] + opt_space = {} + with open(input) as input_json: + opt_points = json.load(input_json) + if 'points' in opt_points: + points = opt_points['points'] + if 'opt_space' in opt_points: + opt_space = opt_points['opt_space'] + return points, opt_space + + +def write_output_points(new_points, output): + with open(args.output, 'w') as output_json: + json.dump(new_points, output_json) + + +def generate_new_points(input_points, opt_space, max_points, num_points): + if len(input_points) >= max_points: + return [] + elif len(input_points) >= max_points - 1: + return [{'point_type': 'stat'}] + + num_points = min(num_points, max_points - 1 - len(input_points)) + + new_points = [] + for _ in range(num_points): + point = {'point_type': 'toy'} + new_points.append(point) + # recommendation = optimizer.provide_recommendation() + return new_points + + +input_points, opt_space = get_input_points(args.input) +new_points = generate_new_points(input_points, opt_space, args.max_points, args.num_points) +write_output_points(new_points, args.output) diff --git a/main/tools/container/docker/toymc/idds_input.json b/main/tools/container/docker/toymc/idds_input.json new file mode 100644 index 00000000..2bc82e60 --- /dev/null +++ b/main/tools/container/docker/toymc/idds_input.json @@ -0,0 +1 @@ +{'opt_space': {}} From 284cce60f77785ea8994ecf6234eb73a5d2bf9b5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:16:30 +0200 Subject: [PATCH 015/156] improve work class --- workflow/lib/idds/workflow/work.py | 32 +++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index bc4eb066..f0a48b7a 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -49,7 +49,7 @@ def get_param_value(self, name): class Collection(Base): - def __init__(self, scope=None, name=None, coll_metadata={}): + def __init__(self, scope=None, name=None, coll_type=CollectionType.Dataset, coll_metadata={}): super(Collection, self).__init__() self.scope = scope self.name = name @@ -59,6 +59,7 @@ def __init__(self, scope=None, name=None, coll_metadata={}): self.internal_id = str(uuid.uuid1()) self.coll_id = None + self.coll_type = coll_type self.status = CollectionStatus.New self.substatus = CollectionStatus.New @@ -88,6 +89,16 @@ def status(self, value): if self.collection: self.collection['status'] = value + @property + def coll_type(self): + return self.get_metadata_item('coll_type', CollectionType.Dataset) + + @coll_type.setter + def coll_type(self, value): + self.add_metadata_item('coll_type', value) + if self.collection: + self.collection['coll_type'] = value + @property def substatus(self): return self.get_metadata_item('substatus', CollectionStatus.New) @@ -110,6 +121,7 @@ def collection(self, value): self.name = self._collection['name'] self.coll_metadata = self._collection['coll_metadata'] self.coll_id = self._collection['coll_id'] + self.coll_type = self._collection['coll_type'] self.status = self._collection['status'] self.substatus = self._collection['substatus'] @@ -1071,7 +1083,13 @@ def add_collection_to_collections(self, coll): coll_metadata = copy.copy(coll) del coll_metadata['scope'] del coll_metadata['name'] - collection = Collection(scope=coll['scope'], name=coll['name'], coll_metadata=coll_metadata) + if 'type' in coll_metadata: + coll_type = coll_metadata['type'] + del coll_metadata['type'] + else: + coll_type = CollectionType.Dataset + + collection = Collection(scope=coll['scope'], name=coll['name'], coll_type=coll_type, coll_metadata=coll_metadata) self.collections[collection.internal_id] = collection return collection @@ -1084,11 +1102,15 @@ def get_primary_input_collection(self): """ *** Function called by Marshaller agent. """ - return self.collections[self.primary_input_collection] + if self.primary_input_collection: + return self.collections[self.primary_input_collection] + return None def add_other_input_collections(self, colls): if not colls: return + if type(colls) not in [list, tuple]: + colls = [colls] for coll in colls: collection = self.add_collection_to_collections(coll) @@ -1191,6 +1213,8 @@ def add_output_collections(self, colls): """ if not colls: return + if type(colls) not in [list, tuple]: + colls = [colls] for coll in colls: collection = self.add_collection_to_collections(coll) @@ -1205,6 +1229,8 @@ def get_output_contents(self): def add_log_collections(self, colls): if not colls: return + if type(colls) not in [list, tuple]: + colls = [colls] for coll in colls: collection = self.add_collection_to_collections(coll) From 24a336e7377fab3809cf098c713dfbda8ab156cb Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:17:44 +0200 Subject: [PATCH 016/156] update work class --- atlas/lib/idds/atlas/workflow/atlasactuatorwork.py | 2 +- atlas/lib/idds/atlas/workflow/atlashpowork.py | 2 +- atlas/lib/idds/atlas/workflow/atlasstageinwork.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py b/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py index e80a125a..8db8de40 100644 --- a/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasactuatorwork.py @@ -453,4 +453,4 @@ def poll_processing_updates(self, processing, input_output_maps): 'output_metadata': processing_outputs}} updated_contents = [] - return update_processing, updated_contents + return update_processing, updated_contents, {} diff --git a/atlas/lib/idds/atlas/workflow/atlashpowork.py b/atlas/lib/idds/atlas/workflow/atlashpowork.py index f27bb3ae..0a34c2c8 100644 --- a/atlas/lib/idds/atlas/workflow/atlashpowork.py +++ b/atlas/lib/idds/atlas/workflow/atlashpowork.py @@ -723,4 +723,4 @@ def poll_processing_updates(self, processing, input_output_maps): update_processing['parameters']['expired_at'] = None processing['expired_at'] = None updated_contents = [] - return update_processing, updated_contents + return update_processing, updated_contents, {} diff --git a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py index 5af9139b..e3d5d01c 100644 --- a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py @@ -373,7 +373,7 @@ def poll_processing_updates(self, processing, input_output_maps): proc = processing['processing_metadata']['processing'] proc.has_new_updates() - return update_processing, updated_contents + return update_processing, updated_contents, {} def get_status_statistics(self, registered_input_output_maps): status_statistics = {} From a9e1cfeea2881f7310397de92743444aa27b9db5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:20:18 +0200 Subject: [PATCH 017/156] update work class --- doma/lib/idds/doma/workflow/domapandawork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index 8737f606..b68bfdaa 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -890,7 +890,7 @@ def poll_processing_updates(self, processing, input_output_maps): (proc.workload_id, str(updated_contents))) self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % (proc.workload_id, str(reactive_contents))) - return update_processing, updated_contents + reactive_contents + return update_processing, updated_contents + reactive_contents, {} def get_status_statistics(self, registered_input_output_maps): status_statistics = {} From 369970578238dc5012e9736c8c75e9dcc235a86e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:23:28 +0200 Subject: [PATCH 018/156] add function to get scope and name --- common/lib/idds/common/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index cf7d42f3..732ef5f1 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -453,3 +453,21 @@ def get_proxy(): def is_new_version(version1, version2): return version1 > version2 + + +def extract_scope_atlas(did, scopes): + # Try to extract the scope from the DSN + if did.find(':') > -1: + if len(did.split(':')) > 2: + raise IDDSException('Too many colons. Cannot extract scope and name') + scope, name = did.split(':')[0], did.split(':')[1] + if name.endswith('/'): + name = name[:-1] + return scope, name + else: + scope = did.split('.')[0] + if did.startswith('user') or did.startswith('group'): + scope = ".".join(did.split('.')[0:2]) + if did.endswith('/'): + did = did[:-1] + return scope, did From 4f4815d1b8fbaa4c1219a848d8dd2ec94ca1374e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:25:34 +0200 Subject: [PATCH 019/156] update carrier to get new contents generated from jobs --- main/lib/idds/agents/carrier/carrier.py | 116 +++++++++++++++++++++++- main/lib/idds/core/processings.py | 4 +- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/main/lib/idds/agents/carrier/carrier.py b/main/lib/idds/agents/carrier/carrier.py index 47f1141e..6e926fc2 100644 --- a/main/lib/idds/agents/carrier/carrier.py +++ b/main/lib/idds/agents/carrier/carrier.py @@ -19,7 +19,8 @@ from idds.common import exceptions from idds.common.constants import (Sections, ProcessingStatus, ProcessingLocking, - MessageStatus) + MessageStatus, ContentStatus, ContentType, + ContentRelationType) from idds.common.utils import setup_logging from idds.core import (transforms as core_transforms, processings as core_processings) @@ -193,6 +194,109 @@ def get_collection_ids(self, collections): coll_ids.append(coll.coll_id) return coll_ids + def get_new_contents(self, processing, new_input_output_maps): + new_input_contents, new_output_contents, new_log_contents = [], [], [] + new_input_dependency_contents = [] + for map_id in new_input_output_maps: + inputs = new_input_output_maps[map_id]['inputs'] if 'inputs' in new_input_output_maps[map_id] else [] + inputs_dependency = new_input_output_maps[map_id]['inputs_dependency'] if 'inputs_dependency' in new_input_output_maps[map_id] else [] + outputs = new_input_output_maps[map_id]['outputs'] if 'outputs' in new_input_output_maps[map_id] else [] + logs = new_input_output_maps[map_id]['logs'] if 'logs' in new_input_output_maps[map_id] else [] + + for input_content in inputs: + content = {'transform_id': processing['transform_id'], + 'coll_id': input_content['coll_id'], + 'request_id': processing['request_id'], + 'workload_id': processing['workload_id'], + 'map_id': map_id, + 'scope': input_content['scope'], + 'name': input_content['name'], + 'min_id': input_content['min_id'] if 'min_id' in input_content else 0, + 'max_id': input_content['max_id'] if 'max_id' in input_content else 0, + 'status': input_content['status'] if 'status' in input_content and input_content['status'] is not None else ContentStatus.New, + 'substatus': input_content['substatus'] if 'substatus' in input_content and input_content['substatus'] is not None else ContentStatus.New, + 'path': input_content['path'] if 'path' in input_content else None, + 'content_type': input_content['content_type'] if 'content_type' in input_content else ContentType.File, + 'content_relation_type': ContentRelationType.Input, + 'bytes': input_content['bytes'], + 'adler32': input_content['adler32'], + 'content_metadata': input_content['content_metadata']} + if content['min_id'] is None: + content['min_id'] = 0 + if content['max_id'] is None: + content['max_id'] = 0 + new_input_contents.append(content) + for input_content in inputs_dependency: + content = {'transform_id': processing['transform_id'], + 'coll_id': input_content['coll_id'], + 'request_id': processing['request_id'], + 'workload_id': processing['workload_id'], + 'map_id': map_id, + 'scope': input_content['scope'], + 'name': input_content['name'], + 'min_id': input_content['min_id'] if 'min_id' in input_content else 0, + 'max_id': input_content['max_id'] if 'max_id' in input_content else 0, + 'status': input_content['status'] if 'status' in input_content and input_content['status'] is not None else ContentStatus.New, + 'substatus': input_content['substatus'] if 'substatus' in input_content and input_content['substatus'] is not None else ContentStatus.New, + 'path': input_content['path'] if 'path' in input_content else None, + 'content_type': input_content['content_type'] if 'content_type' in input_content else ContentType.File, + 'content_relation_type': ContentRelationType.InputDependency, + 'bytes': input_content['bytes'], + 'adler32': input_content['adler32'], + 'content_metadata': input_content['content_metadata']} + if content['min_id'] is None: + content['min_id'] = 0 + if content['max_id'] is None: + content['max_id'] = 0 + new_input_dependency_contents.append(content) + for output_content in outputs: + content = {'transform_id': processing['transform_id'], + 'coll_id': output_content['coll_id'], + 'request_id': processing['request_id'], + 'workload_id': processing['workload_id'], + 'map_id': map_id, + 'scope': output_content['scope'], + 'name': output_content['name'], + 'min_id': output_content['min_id'] if 'min_id' in output_content else 0, + 'max_id': output_content['max_id'] if 'max_id' in output_content else 0, + 'status': ContentStatus.New, + 'substatus': ContentStatus.New, + 'path': output_content['path'] if 'path' in output_content else None, + 'content_type': output_content['content_type'] if 'content_type' in output_content else ContentType.File, + 'content_relation_type': ContentRelationType.Output, + 'bytes': output_content['bytes'], + 'adler32': output_content['adler32'], + 'content_metadata': output_content['content_metadata']} + if content['min_id'] is None: + content['min_id'] = 0 + if content['max_id'] is None: + content['max_id'] = 0 + new_output_contents.append(content) + for log_content in logs: + content = {'transform_id': processing['transform_id'], + 'coll_id': log_content['coll_id'], + 'request_id': processing['request_id'], + 'workload_id': processing['workload_id'], + 'map_id': map_id, + 'scope': log_content['scope'], + 'name': log_content['name'], + 'min_id': log_content['min_id'] if 'min_id' in log_content else 0, + 'max_id': log_content['max_id'] if 'max_id' in log_content else 0, + 'status': ContentStatus.New, + 'substatus': ContentStatus.New, + 'path': log_content['path'] if 'path' in log_content else None, + 'content_type': log_content['content_type'] if 'content_type' in log_content else ContentType.File, + 'content_relation_type': ContentRelationType.Log, + 'bytes': log_content['bytes'], + 'adler32': log_content['adler32'], + 'content_metadata': log_content['content_metadata']} + if content['min_id'] is None: + content['min_id'] = 0 + if content['max_id'] is None: + content['max_id'] = 0 + new_output_contents.append(content) + return new_input_contents + new_output_contents + new_log_contents + new_input_dependency_contents + def process_running_processing(self, processing): try: transform_id = processing['transform_id'] @@ -246,8 +350,8 @@ def process_running_processing(self, processing): # work = processing['processing_metadata']['work'] # outputs = work.poll_processing() - processing_update, content_updates = work.poll_processing_updates(processing, input_output_maps) - + processing_update, content_updates, new_input_output_maps = work.poll_processing_updates(processing, input_output_maps) + new_contents = self.get_new_contents(processing, new_input_output_maps) if processing_update: processing_update['parameters']['locking'] = ProcessingLocking.Idle else: @@ -274,7 +378,8 @@ def process_running_processing(self, processing): processing_update['parameters']['processing_metadata'] = processing['processing_metadata'] ret = {'processing_update': processing_update, - 'content_updates': content_updates} + 'content_updates': content_updates, + 'new_contents': new_contents} except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) @@ -356,7 +461,8 @@ def finish_running_processings(self): # self.logger.info("Main thread finishing running processing %s" % str(processing)) core_processings.update_processing_contents(processing_update=processing.get('processing_update', None), content_updates=processing.get('content_updates', None), - update_messages=processing.get('update_messages', None)) + update_messages=processing.get('update_messages', None), + new_contents=processing.get('new_contents', None)) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) diff --git a/main/lib/idds/core/processings.py b/main/lib/idds/core/processings.py index 97817730..218652cd 100644 --- a/main/lib/idds/core/processings.py +++ b/main/lib/idds/core/processings.py @@ -285,7 +285,7 @@ def update_processing_with_collection_contents(updated_processing, new_processin @transactional_session -def update_processing_contents(processing_update, content_updates, update_messages=None, session=None): +def update_processing_contents(processing_update, content_updates, update_messages=None, new_contents=None, session=None): """ Update processing with contents. @@ -294,6 +294,8 @@ def update_processing_contents(processing_update, content_updates, update_messag """ if content_updates: orm_contents.update_contents(content_updates, session=session) + if new_contents: + orm_contents.add_contents(new_contents, session=session) if processing_update: orm_processings.update_processing(processing_id=processing_update['processing_id'], parameters=processing_update['parameters'], From 7dca6f60e97c081f9b95cc75d5ac18a5fc529be7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:26:24 +0200 Subject: [PATCH 020/156] new atlas panda work --- .../lib/idds/atlas/workflow/atlaspandawork.py | 742 +++++++++++------- 1 file changed, 471 insertions(+), 271 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index 40d381f3..94440524 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -8,185 +8,68 @@ # Authors: # - Wen Guan, , 2020 - 2021 + try: import ConfigParser except ImportError: import configparser as ConfigParser -# try: -# from urllib import quote -# except ImportError: -# from urllib.parse import quote - -import copy +import datetime import os -import re +import random import traceback +from rucio.client.client import Client as RucioClient +from rucio.common.exception import (CannotAuthenticate as RucioCannotAuthenticate) + from idds.common import exceptions -from idds.common.constants import (TransformType, CollectionType, CollectionStatus, - ProcessingStatus, WorkStatus, ContentStatus) +from idds.common.constants import (TransformType, CollectionStatus, CollectionType, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.common.utils import extract_scope_atlas from idds.workflow.work import Work, Processing -from idds.workflow.workflow import Condition - - -class PandaCondition(Condition): - def __init__(self, cond=None, current_work=None, true_work=None, false_work=None): - super(PandaCondition, self).__init__(cond=cond, current_work=current_work, - true_work=true_work, false_work=false_work) +# from idds.workflow.workflow import Condition class ATLASPandaWork(Work): - def __init__(self, executable=None, arguments=None, parameters=None, setup=None, - work_tag='activelearning', exec_type='panda', sandbox=None, work_id=None, + def __init__(self, task_parameters=None, + work_tag='atlas', exec_type='panda', work_id=None, primary_input_collection=None, other_input_collections=None, output_collections=None, log_collections=None, - logger=None, dependency_map=None, task_name="", - panda_task_paramsmap=None): - """ - Init a work/task/transformation. - - :param setup: A string to setup the executable enviroment, it can be None. - :param executable: The executable. - :param arguments: The arguments. - :param parameters: A dict with arguments needed to be replaced. - :param work_type: The work type like data carousel, hyperparameteroptimization and so on. - :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. - :param sandbox: The sandbox. - :param work_id: The work/task id. - :param primary_input_collection: The primary input collection. - :param other_input_collections: List of the input collections. - :param output_collections: List of the output collections. - # :param workflow: The workflow the current work belongs to. - """ - # self.cmd_to_arguments = cmd_to_arguments - self.panda_task_paramsmap = panda_task_paramsmap - self.output_dataset_name = None - super(ATLASPandaWork, self).__init__(executable=executable, arguments=arguments, - parameters=parameters, setup=setup, work_type=TransformType.Processing, - work_tag=work_tag, exec_type=exec_type, sandbox=sandbox, work_id=work_id, + logger=None, + # dependency_map=None, task_name="", + # task_queue=None, processing_type=None, + # prodSourceLabel='test', task_type='test', + # maxwalltime=90000, maxattempt=5, core_count=1, + # encode_command_line=False, + num_retries=5, + # task_log=None, + # task_cloud=None, + # task_rss=0 + ): + + super(ATLASPandaWork, self).__init__(work_type=TransformType.Processing, + work_tag=work_tag, + exec_type=exec_type, + work_id=work_id, primary_input_collection=primary_input_collection, other_input_collections=other_input_collections, output_collections=output_collections, log_collections=log_collections, + release_inputs_after_submitting=True, logger=logger) self.panda_url = None self.panda_url_ssl = None self.panda_monitor = None - self.load_panda_urls() - - # from pandatools import Client - # Client.getTaskParamsMap(23752996) - # (0, '{"buildSpec": {"jobParameters": "-i ${IN} -o ${OUT} --sourceURL ${SURL} -r . ", "archiveName": "sources.0ca6a2fb-4ad0-42d0-979d-aa7c284f1ff7.tar.gz", "prodSourceLabel": "panda"}, "sourceURL": "https://aipanda048.cern.ch:25443", "cliParams": "prun --exec \\"python simplescript.py 0.5 0.5 200 output.json\\" --outDS user.wguan.altest1234 --outputs output.json --nJobs=10", "site": null, "vo": "atlas", "respectSplitRule": true, "osInfo": "Linux-3.10.0-1127.19.1.el7.x86_64-x86_64-with-centos-7.9.2009-Core", "log": {"type": "template", "param_type": "log", "container": "user.wguan.altest1234.log/", "value": "user.wguan.altest1234.log.$JEDITASKID.${SN}.log.tgz", "dataset": "user.wguan.altest1234.log/"}, "transUses": "", "excludedSite": [], "nMaxFilesPerJob": 200, "uniqueTaskName": true, "noInput": true, "taskName": "user.wguan.altest1234/", "transHome": null, "includedSite": null, "nEvents": 10, "nEventsPerJob": 1, "jobParameters": [{"type": "constant", "value": "-j \\"\\" --sourceURL ${SURL}"}, {"type": "constant", "value": "-r ."}, {"padding": false, "type": "constant", "value": "-p \\""}, {"padding": false, "type": "constant", "value": "python%20simplescript.py%200.5%200.5%20200%20output.json"}, {"type": "constant", "value": "\\""}, {"type": "constant", "value": "-l ${LIB}"}, {"container": "user.wguan.altest1234_output.json/", "value": "user.wguan.$JEDITASKID._${SN/P}.output.json", "dataset": "user.wguan.altest1234_output.json/", "param_type": "output", "hidden": true, "type": "template"}, {"type": "constant", "value": "-o \\"{\'output.json\': \'user.wguan.$JEDITASKID._${SN/P}.output.json\'}\\""}], "prodSourceLabel": "user", "processingType": "panda-client-1.4.47-jedi-run", "architecture": "@centos7", "userName": "Wen Guan", "taskType": "anal", "taskPriority": 1000, "countryGroup": "us"}') # noqa E501 - - self.panda_task_id = None - self.init_panda_task_info() - def initialize_work(self): - if not self.is_initialized(): - self.init_new_panda_task_info() - super(ATLASPandaWork, self).initialize_work() + self.task_parameters = None + self.parse_task_parameters(task_parameters) + # self.logger.setLevel(logging.DEBUG) - def get_scope_name(self, dataset): - if dataset.startswith("user"): - scope = "user." + dataset.split('.')[1] - elif dataset.startswith("group"): - scope = "group." + dataset.split('.')[1] - else: - scope = dataset.split('.')[0] - return scope - - def get_output_dataset_name_from_task_paramsmap(self): - if self.panda_task_paramsmap: - cliParams = self.panda_task_paramsmap['cliParams'] - output_dataset_name = cliParams.split("--outDS")[1].strip().split(" ")[0] - return output_dataset_name - return None + self.retry_number = 0 + self.num_retries = num_retries - def init_panda_task_info(self): - if self.panda_task_paramsmap: - self.output_dataset_name = self.get_output_dataset_name_from_task_paramsmap() - self.sandbox = os.path.join(self.panda_task_paramsmap['sourceURL'], 'cache/' + self.panda_task_paramsmap['buildSpec']['archiveName']) - for p in self.panda_task_paramsmap["jobParameters"]: - if 'param_type' in p and p['param_type'] == 'output': - output_dataset = p['dataset'] - output_dataset = output_dataset.replace("/", "") - scope = self.get_scope_name(output_dataset) - primary_input_collection = {'scope': scope, 'name': output_dataset} - output_collection = {'scope': scope, 'name': output_dataset} - self.set_primary_input_collection(primary_input_collection) - self.add_output_collections([output_collection]) - if 'log' in p: - log_dataset = p['dataset'] - log_dataset = log_dataset.replace("/", "") - scope = self.get_scope_name(log_dataset) - log_collection = {'scope': scope, 'name': log_dataset} - self.add_log_collections([log_collection]) - - def init_new_panda_task_info(self): - if not self.panda_task_paramsmap: - return - - # generate new dataset name - # self.padding = self.sequence_in_workflow - new_dataset_name = self.output_dataset_name + "_" + str(self.sequence_id) - for coll_id in self.collections: - coll = self.collections[coll_id] - coll['name'] = coll['name'].replace(self.output_dataset_name, new_dataset_name) - - self.panda_task_paramsmap['cliParams'] = \ - self.panda_task_paramsmap['cliParams'].replace(self.output_dataset_name, new_dataset_name) - - self.panda_task_paramsmap['taskName'] = \ - self.panda_task_paramsmap['taskName'].replace(self.output_dataset_name, new_dataset_name) - - jobParameters = self.panda_task_paramsmap['jobParameters'] - for p in jobParameters: - if 'container' in p: - p['container'] = p['container'].replace(self.output_dataset_name, new_dataset_name) - if 'dataset' in p: - p['dataset'] = p['dataset'].replace(self.output_dataset_name, new_dataset_name) - - log = self.panda_task_paramsmap['log'] - if 'value' in log: - log['value'] = log['value'].replace(self.output_dataset_name, new_dataset_name) - if 'container' in log: - log['container'] = log['container'].replace(self.output_dataset_name, new_dataset_name) - if 'dataset' in log: - log['dataset'] = log['dataset'].replace(self.output_dataset_name, new_dataset_name) - - self.parse_arguments() - - def parse_arguments(self): - try: - # arguments = self.get_arguments() - # parameters = self.get_parameters() - new_parameters = self.get_parameters() - - if new_parameters: - self.panda_task_paramsmap['cliParams'] = self.panda_task_paramsmap['cliParams'].format(**new_parameters) - - # todo - # jobParameters = self.panda_task_paramsmap['jobParameters'] - # for p in jobParameters: - # if 'value' in p: - # p['value'] = p['value'].replace(quote(arguments), quote(new_arguments)) - - # return new_arguments - except Exception as ex: - self.add_errors(str(ex)) - - def generate_work_from_template(self): - new_work = super(ATLASPandaWork, self).generate_work_from_template() - # new_work.unset_initialized() - # new_work.panda_task_id = None - return new_work - - def set_parameters(self, parameters): - self.parameters = parameters - # trigger to submit new tasks - self.unset_initialized() - self.panda_task_id = None + self.load_panda_urls() def my_condition(self): if self.is_finished(): @@ -210,7 +93,7 @@ def load_panda_config(self): def load_panda_urls(self): panda_config = self.load_panda_config() - self.logger.debug("panda config: %s" % panda_config) + # self.logger.debug("panda config: %s" % panda_config) self.panda_url = None self.panda_url_ssl = None self.panda_monitor = None @@ -219,60 +102,146 @@ def load_panda_urls(self): if panda_config.has_option('panda', 'panda_monitor_url'): self.panda_monitor = panda_config.get('panda', 'panda_monitor_url') os.environ['PANDA_MONITOR_URL'] = self.panda_monitor - self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) + # self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) if panda_config.has_option('panda', 'panda_url'): self.panda_url = panda_config.get('panda', 'panda_url') os.environ['PANDA_URL'] = self.panda_url - self.logger.debug("Panda url: %s" % str(self.panda_url)) + # self.logger.debug("Panda url: %s" % str(self.panda_url)) if panda_config.has_option('panda', 'panda_url_ssl'): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl - self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']: self.panda_monitor = os.environ['PANDA_MONITOR_URL'] - self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) + # self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) if not self.panda_url and 'PANDA_URL' in os.environ and os.environ['PANDA_URL']: self.panda_url = os.environ['PANDA_URL'] - self.logger.debug("Panda url: %s" % str(self.panda_url)) + # self.logger.debug("Panda url: %s" % str(self.panda_url)) if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] - self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + + def set_agent_attributes(self, attrs, req_attributes=None): + if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: + attrs['life_time'] = None + super(ATLASPandaWork, self).set_agent_attributes(attrs) + if self.agent_attributes and 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: + self.num_retries = int(self.agent_attributes['num_retries']) + + def parse_task_parameters(self, task_parameters): + if self.task_parameters: + return + elif not task_parameters: + return + self.task_parameters = task_parameters + + try: + if 'taskName' in self.task_parameters: + self.task_name = self.task_parameters['taskName'] + self.set_work_name(self.task_name) + + if 'jobParameters' in self.task_parameters: + jobParameters = self.task_parameters['jobParameters'] + for jobPs in jobParameters: + if type(jobPs) in [tuple, list]: + for jobP in jobPs: + if type(jobP) in [dict]: + if 'dataset' in jobP and 'param_type' in jobP: + if jobP['param_type'] == 'input': + input_c = jobP['dataset'] + scope, name = extract_scope_atlas(input_c, scopes=[]) + input_coll = {'scope': scope, 'name': name} + self.set_primary_input_collection(input_coll) + if jobP['param_type'] == 'output': + output_c = jobP['dataset'] + scope, name = extract_scope_atlas(output_c, scopes=[]) + output_coll = {'scope': scope, 'name': name} + self.add_output_collections([output_coll]) + + if 'log' in self.task_parameters: + log = self.task_parameters['log'] + dataset = log['dataset'] + scope, name = extract_scope_atlas(dataset, scopes=[]) + log_col = {'scope': scope, 'name': name} + self.add_log_collections(log_col) + + if not self.get_primary_input_collection(): + output_colls = self.get_output_collections() + output_coll = output_colls[0] + name = 'pseudo_input.' + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + input_coll = {'scope': output_coll.scope, 'name': name, 'type': CollectionType.PseudoDataset} + self.set_primary_input_collection(input_coll) + + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + self.add_errors(str(ex)) + + def get_rucio_client(self): + try: + client = RucioClient() + except RucioCannotAuthenticate as error: + self.logger.error(error) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc())) + return client def poll_external_collection(self, coll): try: - # if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: if coll.status in [CollectionStatus.Closed]: return coll else: - # client = self.get_rucio_client() - # did_meta = client.get_metadata(scope=coll['scope'], name=coll['name']) - coll.coll_metadata['bytes'] = 0 - coll.coll_metadata['total_files'] = 0 - coll.coll_metadata['availability'] = True - coll.coll_metadata['events'] = 0 - coll.coll_metadata['is_open'] = False - coll.coll_metadata['run_number'] = None - coll.coll_metadata['did_type'] = 'DATASET' - coll.coll_metadata['list_all_files'] = False - - if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: - coll_status = CollectionStatus.Closed - else: - coll_status = CollectionStatus.Open - coll.status = coll_status - - if 'did_type' in coll.coll_metadata: - if coll.coll_metadata['did_type'] == 'DATASET': - coll_type = CollectionType.Dataset - elif coll.coll_metadata['did_type'] == 'CONTAINER': - coll_type = CollectionType.Container + if coll.coll_type == CollectionType.PseudoDataset: + coll.coll_metadata['bytes'] = 0 + coll.coll_metadata['total_files'] = 0 + coll.coll_metadata['availability'] = True + coll.coll_metadata['events'] = 0 + coll.coll_metadata['is_open'] = False + coll.coll_metadata['run_number'] = None + coll.coll_metadata['did_type'] = 'DATASET' + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed else: - coll_type = CollectionType.File - else: - coll_type = CollectionType.Dataset - coll.coll_metadata['coll_type'] = coll_type + coll_status = CollectionStatus.Open + coll.status = coll_status + + coll.coll_metadata['coll_type'] = coll.coll_type + return coll + else: + client = self.get_rucio_client() + did_meta = client.get_metadata(scope=coll.scope, name=coll.name) + + coll.coll_metadata['bytes'] = did_meta['bytes'] + coll.coll_metadata['total_files'] = did_meta['length'] + coll.coll_metadata['availability'] = did_meta['availability'] + coll.coll_metadata['events'] = did_meta['events'] + coll.coll_metadata['is_open'] = did_meta['is_open'] + coll.coll_metadata['run_number'] = did_meta['run_number'] + coll.coll_metadata['did_type'] = did_meta['did_type'] + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + if 'did_type' in coll.coll_metadata: + if coll.coll_metadata['did_type'] == 'DATASET': + coll_type = CollectionType.Dataset + elif coll.coll_metadata['did_type'] == 'CONTAINER': + coll_type = CollectionType.Container + else: + coll_type = CollectionType.File + else: + coll_type = CollectionType.Dataset + coll.coll_metadata['coll_type'] = coll_type + coll.coll_type = coll_type return coll except Exception as ex: self.logger.error(ex) @@ -286,6 +255,10 @@ def get_input_collections(self): colls = [self.primary_input_collection] + self.other_input_collections for coll_int_id in colls: coll = self.collections[coll_int_id] + # if self.is_internal_collection(coll): + # coll = self.poll_internal_collection(coll) + # else: + # coll = self.poll_external_collection(coll) coll = self.poll_external_collection(coll) self.collections[coll_int_id] = coll return super(ATLASPandaWork, self).get_input_collections() @@ -315,8 +288,44 @@ def get_mapped_inputs(self, mapped_input_output_maps): ret.append(primary_input) return ret + def get_mapped_outputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + outputs = mapped_input_output_maps[map_id]['outputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_output = outputs[0] + for ip in outputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_output = ip + ret.append(primary_output) + return ret + + def map_file_to_content(self, coll_id, scope, name): + content = {'coll_id': coll_id, + 'scope': scope, + 'name': name, # or a different file name from the dataset name + 'bytes': 1, + 'adler32': '12345678', + 'min_id': 0, + 'max_id': 1, + 'content_type': ContentType.File, + # 'content_relation_type': content_relation_type, + # here events is all events for eventservice, not used here. + 'content_metadata': {'events': 1}} + return content + + def get_next_map_id(self, input_output_maps): + mapped_keys = input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + return next_key + def get_new_input_output_maps(self, mapped_input_output_maps={}): """ + *** Function called by Transformer agent. New inputs which are not yet mapped to outputs. :param mapped_input_output_maps: Inputs that are already mapped. @@ -333,29 +342,22 @@ def get_new_input_output_maps(self, mapped_input_output_maps={}): new_inputs.append(ip) # to avoid cheking new inputs if there are no new inputs anymore - if (not new_inputs and 'status' in self.collections[self.primary_input_collection] - and self.collections[self.primary_input_collection]['status'] in [CollectionStatus.Closed]): # noqa: W503 + if (not new_inputs and self.collections[self.primary_input_collection].status in [CollectionStatus.Closed]): # noqa: W503 self.set_has_new_inputs(False) else: - mapped_keys = mapped_input_output_maps.keys() - if mapped_keys: - next_key = max(mapped_keys) + 1 - else: - next_key = 1 - for ip in new_inputs: - out_ip = copy.deepcopy(ip) - ip['status'] = ContentStatus.Available - ip['substatus'] = ContentStatus.Available - out_ip['coll_id'] = self.collections[self.output_collections[0]]['coll_id'] - new_input_output_maps[next_key] = {'inputs': [ip], - 'outputs': [out_ip], - 'inputs_dependency': [], - 'logs': []} - next_key += 1 + pass + # self.logger.debug("get_new_input_output_maps, new_input_output_maps: %s" % str(new_input_output_maps)) + self.logger.debug("get_new_input_output_maps, new_input_output_maps len: %s" % len(new_input_output_maps)) return new_input_output_maps - def get_processing(self, input_output_maps, without_creating=False): + def use_dependency_to_release_jobs(self): + """ + *** Function called by Transformer agent. + """ + return False + + def get_processing(self, input_output_maps=[], without_creating=False): """ *** Function called by Transformer agent. @@ -366,6 +368,7 @@ def get_processing(self, input_output_maps, without_creating=False): return self.processings[self.active_processings[0]] else: if not without_creating: + # return None return self.create_processing(input_output_maps) return None @@ -375,9 +378,9 @@ def create_processing(self, input_output_maps=[]): :param input_output_maps: new maps from inputs to outputs. """ - processing_metadata = {'panda_task_id': self.panda_task_id} + processing_metadata = {'task_param': self.task_parameters} proc = Processing(processing_metadata=processing_metadata) - proc.workload_id = self.panda_task_id + proc.workload_id = None self.add_processing_to_processings(proc) self.active_processings.append(proc.internal_id) return proc @@ -386,48 +389,154 @@ def submit_panda_task(self, processing): try: from pandatools import Client - status, tmpOut = Client.insertTaskParams(self.panda_task_paramsmap, False, True) - if status == 0: - tmp_status, tmp_output = tmpOut - m = re.search("jediTaskID=(\d+)", tmp_output) # noqa W605 - task_id = int(m.group(1)) - processing.workload_id = task_id + proc = processing['processing_metadata']['processing'] + task_param = proc.processing_metadata['task_param'] + return_code = Client.insertTaskParams(task_param, verbose=True) + if return_code[0] == 0: + return return_code[1][1] else: - self.add_errors(tmpOut) - raise Exception(tmpOut) + self.logger.warn("submit_panda_task, return_code: %s" % str(return_code)) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + # raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc())) + return None def submit_processing(self, processing): """ *** Function called by Carrier agent. """ - if 'panda_task_id' in processing['processing_metadata'] and processing['processing_metadata']['panda_task_id']: + proc = processing['processing_metadata']['processing'] + if proc.workload_id: + # if 'task_id' in processing['processing_metadata'] and processing['processing_metadata']['task_id']: pass else: - self.set_user_proxy() - self.submit_panda_task(processing) - self.unset_user_proxy() - - def poll_panda_task(self, processing): - if 'panda_task_id' in processing['processing_metadata']: + task_id = self.submit_panda_task(processing) + # processing['processing_metadata']['task_id'] = task_id + # processing['processing_metadata']['workload_id'] = task_id + proc.workload_id = task_id + if task_id: + proc.submitted_at = datetime.datetime.utcnow() + + def poll_panda_task_status(self, processing): + if 'processing' in processing['processing_metadata']: from pandatools import Client - status, task_status = Client.getTaskStatus(processing.workload_id) + proc = processing['processing_metadata']['processing'] + status, task_status = Client.getTaskStatus(proc.workload_id) if status == 0: return task_status else: return 'failed' return None + def get_processing_status_from_panda_status(self, task_status): + if task_status in ['registered', 'defined', 'assigning']: + processing_status = ProcessingStatus.Submitting + elif task_status in ['ready', 'pending', 'scouting', 'scouted', 'prepared', 'topreprocess', 'preprocessing']: + processing_status = ProcessingStatus.Submitted + elif task_status in ['running', 'toretry', 'toincexec', 'throttled']: + processing_status = ProcessingStatus.Running + elif task_status in ['done']: + processing_status = ProcessingStatus.Finished + elif task_status in ['finished', 'paused']: + # finished, finishing, waiting it to be done + processing_status = ProcessingStatus.SubFinished + elif task_status in ['failed', 'aborted', 'broken', 'exhausted']: + # aborting, tobroken + processing_status = ProcessingStatus.Failed + else: + # finished, finishing, aborting, topreprocess, preprocessing, tobroken + # toretry, toincexec, rerefine, paused, throttled, passed + processing_status = ProcessingStatus.Submitted + return processing_status + + def get_panda_task_id(self, processing): + from pandatools import Client + + start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=10) + start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + status, results = Client.getJobIDsJediTasksInTimeRange(start_time, task_type=self.task_type, verbose=False) + if status != 0: + self.logger.warn("Error to poll latest tasks in last ten hours: %s, %s" % (status, results)) + return None + + proc = processing['processing_metadata']['processing'] + task_id = None + for req_id in results: + task_name = results[req_id]['taskName'] + if proc.workload_id is None and task_name == self.task_name: + task_id = results[req_id]['jediTaskID'] + # processing['processing_metadata']['task_id'] = task_id + # processing['processing_metadata']['workload_id'] = task_id + proc.workload_id = task_id + if task_id: + proc.submitted_at = datetime.datetime.utcnow() + + return task_id + + def poll_panda_task(self, processing=None, input_output_maps=None): + task_id = None + try: + from pandatools import Client + + if processing: + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + if task_id is None: + task_id = self.get_panda_task_id(processing) + + if task_id: + # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) + self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) + if task_info[0] != 0: + self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) + return ProcessingStatus.Submitting, {} + + task_info = task_info[1] + + processing_status = self.get_processing_status_from_panda_status(task_info["status"]) + + if processing_status in [ProcessingStatus.SubFinished]: + if self.retry_number < self.num_retries: + self.reactivate_processing(processing) + processing_status = ProcessingStatus.Submitted + self.retry_number += 1 + + return processing_status, [], {} + else: + return ProcessingStatus.Failed, [], {} + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + self.logger.error(msg) + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException(msg) + return ProcessingStatus.Submitting, [], {} + def kill_processing(self, processing): try: if processing: from pandatools import Client - task_id = processing.workload_id + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + # task_id = processing['processing_metadata']['task_id'] + # Client.killTask(task_id) + Client.finishTask(task_id, soft=False) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def kill_processing_force(self, processing): + try: + if processing: + from pandatools import Client + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + # task_id = processing['processing_metadata']['task_id'] Client.killTask(task_id) + # Client.finishTask(task_id, soft=True) except Exception as ex: msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) raise exceptions.IDDSException(msg) @@ -436,8 +545,13 @@ def reactivate_processing(self, processing): try: if processing: from pandatools import Client - task_id = processing.workload_id - Client.retryTask(task_id) + # task_id = processing['processing_metadata']['task_id'] + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + + # Client.retryTask(task_id) + status, out = Client.retryTask(task_id, newParams={}) + self.logger.warn("Retry processing(%s) with task id(%s): %s, %s" % (processing['processing_id'], task_id, status, out)) # Client.reactivateTask(task_id) # Client.resumeTask(task_id) except Exception as ex: @@ -451,63 +565,149 @@ def poll_processing_updates(self, processing, input_output_maps): updated_contents = [] update_processing = {} reset_expired_at = False + reactive_contents = [] + # self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) if processing: - if self.tocancel: - self.logger.info("Cancelling processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], processing['processing_metadata']['task_id'])) - self.kill_processing(processing) - self.tocancel = False - elif self.tosuspend: - self.logger.info("Suspending processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], processing['processing_metadata']['task_id'])) - self.kill_processing(processing) - self.tosuspend = False - elif self.toresume: - self.logger.info("Resuming processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], processing['processing_metadata']['task_id'])) + proc = processing['processing_metadata']['processing'] + if proc.tocancel: + self.logger.info("Cancelling processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tocancel = False + proc.polling_retries = 0 + elif proc.tosuspend: + self.logger.info("Suspending processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tosuspend = False + proc.polling_retries = 0 + elif proc.toresume: + self.logger.info("Resuming processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) self.reactivate_processing(processing) - self.toresume = False reset_expired_at = True - elif self.toexpire: - self.logger.info("Expiring processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], processing['processing_metadata']['task_id'])) + proc.toresume = False + proc.polling_retries = 0 + proc.has_new_updates() + # reactive_contents = self.reactive_contents(input_output_maps) + # elif self.is_processing_expired(processing): + elif proc.toexpire: + self.logger.info("Expiring processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.toexpire = False + proc.polling_retries = 0 + elif proc.tofinish or proc.toforcefinish: + self.logger.info("Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.tofinish = False + proc.toforcefinish = False + proc.polling_retries = 0 + elif self.is_all_contents_terminated_and_with_missing(input_output_maps): + self.logger.info("All contents terminated(There are Missing contents). Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) self.kill_processing(processing) - task_status = self.poll_panda_task(processing) - if task_status: - if task_status in ['registered', 'defined']: - processing_status = ProcessingStatus.Submitted - elif task_status in ['assigning', 'ready', 'pending', 'scouting', 'scouted', 'running', 'prepared']: - processing_status = ProcessingStatus.Running - elif task_status in ['done']: - # finished, finishing, waiting it to be done - processing_status = ProcessingStatus.Finished - elif task_status in ['failed', 'aborted', 'broken', 'exhausted']: - processing_status = ProcessingStatus.Failed - else: - # finished, finishing, aborting, topreprocess, preprocessing, tobroken - # toretry, toincexec, rerefine, paused, throttled, passed - processing_status = ProcessingStatus.Running - - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': processing_status}} - if reset_expired_at: - update_processing['parameters']['expired_at'] = None - processing['expired_at'] = None - if (processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] - or processing['status'] in [ProcessingStatus.Resuming]): # noqa W503 - update_processing['parameters']['status'] = ProcessingStatus.Resuming + processing_status, poll_updated_contents, new_input_output_maps = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) + self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) + self.logger.debug("poll_processing_updates, update_contents: %s" % str(poll_updated_contents)) + + if poll_updated_contents: + proc.has_new_updates() + for content in poll_updated_contents: + updated_content = {'content_id': content['content_id'], + 'substatus': content['substatus'], + 'content_metadata': content['content_metadata']} + updated_contents.append(updated_content) + + content_substatus = {'finished': 0, 'unfinished': 0} + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if content.get('substatus', ContentStatus.New) != ContentStatus.Available: + content_substatus['unfinished'] += 1 + else: + content_substatus['finished'] += 1 - return update_processing, updated_contents + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] and updated_contents: + self.logger.info("Processing %s is terminated, but there are still contents to be flushed. Waiting." % (proc.workload_id)) + # there are still polling contents, should not terminate the task. + processing_status = ProcessingStatus.Running - def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}): - # self.syn_collection_status() + if processing_status in [ProcessingStatus.SubFinished] and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + # found that a 'done' panda task has got a 'finished' status. Maybe in this case 'finished' is a transparent status. + if proc.polling_retries is None: + proc.polling_retries = 0 - if self.is_processings_terminated() and not self.has_new_inputs(): - if not self.is_all_outputs_flushed(registered_input_output_maps): - self.logger.warn("The processing is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform") + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if proc.polling_retries is not None and proc.polling_retries < 3: + self.logger.info("processing %s polling_retries(%s) < 3, keep running" % (processing['processing_id'], proc.polling_retries)) + processing_status = ProcessingStatus.Running + proc.polling_retries += 1 + else: + proc.polling_retries = 0 + + if proc.in_operation_time(): + processing_status = ProcessingStatus.Running + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': processing_status}} + if reset_expired_at: + processing['expired_at'] = None + update_processing['parameters']['expired_at'] = None + proc.polling_retries = 0 + # if (processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] + # or processing['status'] in [ProcessingStatus.Resuming]): # noqa W503 + # using polling_retries to poll it again when panda may update the status in a delay(when issuing retryTask, panda will not update it without any delay). + update_processing['parameters']['status'] = ProcessingStatus.Resuming + proc.status = update_processing['parameters']['status'] + + self.logger.debug("poll_processing_updates, task: %s, update_processing: %s" % + (proc.workload_id, str(update_processing))) + self.logger.debug("poll_processing_updates, task: %s, updated_contents: %s" % + (proc.workload_id, str(updated_contents))) + self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % + (proc.workload_id, str(reactive_contents))) + return update_processing, updated_contents + reactive_contents, new_input_output_maps + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + for map_id in registered_input_output_maps: + outputs = registered_input_output_maps[map_id]['outputs'] + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + self.status_statistics = status_statistics + self.logger.debug("registered_input_output_maps, status_statistics: %s" % str(status_statistics)) + return status_statistics + + def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + super(ATLASPandaWork, self).syn_work_status(registered_input_output_maps, all_updates_flushed, output_statistics, to_release_input_contents) + # self.get_status_statistics(registered_input_output_maps) + self.status_statistics = output_statistics + + self.logger.debug("syn_work_status, self.active_processings: %s" % str(self.active_processings)) + self.logger.debug("syn_work_status, self.has_new_inputs(): %s" % str(self.has_new_inputs)) + self.logger.debug("syn_work_status, coll_metadata_is_open: %s" % + str(self.collections[self.primary_input_collection].coll_metadata['is_open'])) + self.logger.debug("syn_work_status, primary_input_collection_status: %s" % + str(self.collections[self.primary_input_collection].status)) + + self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) + self.logger.debug("syn_work_status(%s): is_input_collections_closed: %s" % (str(self.get_processing_ids()), str(self.is_input_collections_closed()))) + self.logger.debug("syn_work_status(%s): has_new_inputs: %s" % (str(self.get_processing_ids()), str(self.has_new_inputs))) + self.logger.debug("syn_work_status(%s): has_to_release_inputs: %s" % (str(self.get_processing_ids()), str(self.has_to_release_inputs()))) + self.logger.debug("syn_work_status(%s): to_release_input_contents: %s" % (str(self.get_processing_ids()), str(to_release_input_contents))) + + if self.is_processings_terminated() and self.is_input_collections_closed() and not self.has_new_inputs and not self.has_to_release_inputs() and not to_release_input_contents: + # if not self.is_all_outputs_flushed(registered_input_output_maps): + if not all_updates_flushed: + self.logger.warn("The work processings %s is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform" % str(self.get_processing_ids())) return if self.is_processings_finished(): self.status = WorkStatus.Finished - elif self.is_processings_failed(): + if self.is_processings_failed(): self.status = WorkStatus.Failed elif self.is_processings_subfinished(): self.status = WorkStatus.SubFinished From cadbdce38c2232405f5cff316e1007b0a2296dcc Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:27:37 +0200 Subject: [PATCH 021/156] new test --- main/lib/idds/tests/core_tests.py | 6 +- .../tests/resume_subfinished_data_carousel.sh | 2 + main/lib/idds/tests/run_sql.py | 2 +- main/lib/idds/tests/test_atlaspandawork.py | 116 ++++++++++++++++++ main/lib/idds/tests/test_hyperparameteropt.py | 3 + 5 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 main/lib/idds/tests/test_atlaspandawork.py diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 599b3355..b4b97d4a 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -77,9 +77,7 @@ def release_inputs_test(): print(update_contents) -release_inputs_test() - -sys.exit(0) +# release_inputs_test() def show_works(req): @@ -104,7 +102,7 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=118, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=132, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh index 2eeca111..b3db9802 100644 --- a/main/lib/idds/tests/resume_subfinished_data_carousel.sh +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -2,6 +2,8 @@ request_ids=(75039 75065 75569 75683 76087 76181 76211 76455 76727 77281 77283 77285 77287 77385 77419 77643 78177 78241 78497 78517 78533 78535 78743 78745 78787 78895 78947 79037 79045 79057 79065 79147 79155 79161 79179 79191 79207 79263 79297 79317 79337 79347 79395 79401 79463 79465 79525 79563 79737 79743 79755 79759 79967 79975 79977 79987 80001 80083 80087 80099 80107 80109 80123 80131 80211 80215 80217 80257 80291 80339 80341 80433 80565 80635 80705 80747 80883 80887 80923 81015 81017 81033 81035 81065 81231 81237 81303 81307 81309 81379 81547 82169) +request_ids=(75039 77281 77283 77285 77287 83271 83483 77279 77367 77847 77947 78123 78205 80547 80827 82153) + for request_id in "${request_ids[@]}"; do echo idds resume_requests --request_id=${request_id} idds resume_requests --request_id=${request_id} diff --git a/main/lib/idds/tests/run_sql.py b/main/lib/idds/tests/run_sql.py index 813a5437..ac327154 100644 --- a/main/lib/idds/tests/run_sql.py +++ b/main/lib/idds/tests/run_sql.py @@ -23,7 +23,7 @@ def get_subfinished_requests(db_pool): connection = db_pool.acquire() req_ids = [] - sql = """select request_id from atlas_IDDS.requests where status=4 and scope!='hpo'""" + sql = """select request_id from atlas_IDDS.requests where status in (4,5) and scope!='hpo'""" cursor = connection.cursor() cursor.execute(sql) rows = cursor.fetchall() diff --git a/main/lib/idds/tests/test_atlaspandawork.py b/main/lib/idds/tests/test_atlaspandawork.py new file mode 100644 index 00000000..10f5b109 --- /dev/null +++ b/main/lib/idds/tests/test_atlaspandawork.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2019 - 2021 + + +""" +Test client. +""" + + +from idds.client.clientmanager import ClientManager +from idds.common.utils import get_rest_host + + +def get_workflow(): + from idds.workflow.workflow import Workflow + from idds.atlas.workflow.atlaspandawork import ATLASPandaWork + + task_parameters = {"architecture": "", + "cliParams": "prun --exec 'echo %RNDM:10 > seed.txt' --outputs seed.txt --nJobs 3 --outDS user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top", + "excludedSite": [], + "includedSite": None, + "jobParameters": [ + { + "type": "constant", + "value": "-j \"\" --sourceURL ${SURL}" + }, + { + "type": "constant", + "value": "-r ." + }, + { + "padding": False, + "type": "constant", + "value": "-p \"" + }, + { + "padding": False, + "type": "constant", + "value": "echo%20%25RNDM%3A10%20%3E%20seed.txt" + }, + { + "type": "constant", + "value": "\"" + }, + { + "type": "constant", + "value": "-a jobO.6f1dc5a8-eeb3-4cdf-aed0-0930b6a0e815.tar.gz" + }, + [ + { + "container": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", + "dataset": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", + "hidden": True, + "param_type": "output", + "type": "template", + "value": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt" + } + ], + { + "type": "constant", + "value": "-o \"{'seed.txt': 'user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt'}\"" + } + ], + "log": { + "container": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top/", + "dataset": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top/", + "param_type": "log", + "type": "template", + "value": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.log.$JEDITASKID.${SN}.log.tgz" + }, + "nEvents": 3, + "nEventsPerJob": 1, + "nMaxFilesPerJob": 200, + "noInput": True, + "osInfo": "Linux-3.10.0-1160.31.1.el7.x86_64-x86_64-with-centos-7.9.2009-Core", + "processingType": "panda-client-1.4.79-jedi-run", + "prodSourceLabel": "user", + "respectSplitRule": True, + "site": None, + "sourceURL": "https://aipanda059.cern.ch:25443", + "taskName": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top", + "transHome": None, + "transUses": "", + "uniqueTaskName": True, + "userName": "Tadashi Maeno", + "vo": "atlas" + } + + work = ATLASPandaWork(task_parameters=task_parameters) + wf = Workflow() + wf.set_workload_id(234567) + wf.add_work(work) + return wf + + +if __name__ == '__main__': + host = get_rest_host() + # props = get_req_properties() + # props = get_example_real_tape_stagein_request() + # props = get_example_prodsys2_tape_stagein_request() + # props = get_example_active_learning_request() + workflow = get_workflow() + + # props = pre_check(props) + # print(props) + + wm = ClientManager(host=host) + request_id = wm.submit(workflow) + print(request_id) diff --git a/main/lib/idds/tests/test_hyperparameteropt.py b/main/lib/idds/tests/test_hyperparameteropt.py index f0bd0db2..da0f6cd8 100644 --- a/main/lib/idds/tests/test_hyperparameteropt.py +++ b/main/lib/idds/tests/test_hyperparameteropt.py @@ -28,6 +28,9 @@ def get_workflow(): # request_metadata for docker method request_metadata = {'workload_id': '20525134', 'sandbox': 'wguanicedew/idds_hpo_nevergrad', 'workdir': '/data', 'executable': 'docker', 'arguments': 'python /opt/hyperparameteropt_nevergrad.py --max_points=%MAX_POINTS --num_points=%NUM_POINTS --input=/data/%IN --output=/data/%OUT', 'output_json': 'output.json', 'opt_space': {"A": {"type": "Choice", "params": {"choices": [1, 4]}}, "B": {"type": "Scalar", "bounds": [0, 5]}}, 'initial_points': [({'A': 1, 'B': 2}, 0.3), ({'A': 1, 'B': 3}, None)], 'max_points': 20, 'num_points_per_generation': 10} # noqa E501 + # request_metadata for docker toymc method + request_metadata = {'workload_id': '20525147', 'sandbox': 'wguanicedew/idds_hpo_toymc', 'workdir': '/data', 'executable': 'docker', 'arguments': 'python /opt/hyperparameteropt_toymc.py --max_points=%MAX_POINTS --num_points=%NUM_POINTS --input=/data/%IN --output=/data/%OUT', 'output_json': 'output.json', 'opt_space': {"A": {}}, 'initial_points': [], 'max_points': 20, 'num_points_per_generation': 10} # noqa E501 + work = ATLASHPOWork(executable=request_metadata.get('executable', None), arguments=request_metadata.get('arguments', None), parameters=request_metadata.get('parameters', None), From 2d77853ae4d12b95acb0b100c5c9221d3b332da6 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 14:35:50 +0200 Subject: [PATCH 022/156] new version 0.6.0 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 +-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 +-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 +-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 ++-- monitor/conf.js | 12 ++++---- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/lib/idds/workflow/workflow.py | 40 ++++++++++++++++++++++++++ workflow/tools/env/environment.yml | 2 +- 15 files changed, 64 insertions(+), 24 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index b025b4fa..aeed08aa 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 008df559..8cbd4d20 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.5.4 - - idds-workflow==0.5.4 \ No newline at end of file + - idds-common==0.6.0 + - idds-workflow==0.6.0 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index b025b4fa..aeed08aa 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index fc9d2746..241fe2e4 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.5.4 - - idds-workflow==0.5.4 \ No newline at end of file + - idds-common==0.6.0 + - idds-workflow==0.6.0 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index b025b4fa..aeed08aa 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 9cccd94e..7532bfb8 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 7ec1bdf9..86ca910a 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.5.4 - - idds-workflow==0.5.4 \ No newline at end of file + - idds-common==0.6.0 + - idds-workflow==0.6.0 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index b025b4fa..aeed08aa 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 07aa3f84..e62ae9d4 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.5.4 - - idds-workflow==0.5.4 - - idds-client==0.5.4 \ No newline at end of file + - idds-common==0.6.0 + - idds-workflow==0.6.0 + - idds-client==0.6.0 \ No newline at end of file diff --git a/monitor/conf.js b/monitor/conf.js index e3a91e83..a212aad7 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus750.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus750.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus750.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus750.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus750.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus750.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus765.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus765.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus765.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus765.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus765.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus765.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/version.py b/monitor/version.py index b025b4fa..aeed08aa 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/website/version.py b/website/version.py index b025b4fa..aeed08aa 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index b025b4fa..aeed08aa 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.5.4" +release_version = "0.6.0" diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index aa4c5c50..89b2cd71 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -16,6 +16,8 @@ import time import uuid +from enum import Enum + from idds.common.utils import json_dumps, setup_logging, get_proxy from idds.common.utils import str_to_date from .base import Base @@ -78,6 +80,16 @@ def get_cond_status(self): else: return False + def is_condition_true(self): + if self.get_cond_status(): + return True + return False + + def is_condition_false(self): + if not self.get_cond_status(): + return True + return False + def get_next_work(self): if self.get_cond_status(): return self.true_work @@ -85,6 +97,34 @@ def get_next_work(self): return self.false_work +class ConditionOperator(Enum): + And = 0 + Or = 1 + + +class MultiCondition(Condition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + super(MultiCondition, self).__init__(cond=cond, current_work=current_work, true_work=true_work, + false_work=false_work, logger=logger) + self.additional_conds = [] + + def add_condition(self, cond, current_work, operator=ConditionOperator.And): + cond = {'cond': cond, 'current_work': current_work.get_template_id(), 'operator': operator} + self.additional_conds.append(cond) + + def get_cond_status(self): + if callable(self.cond): + if self.cond(): + return True + else: + return False + else: + if self.cond: + return True + else: + return False + + class Workflow(Base): def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 3270efa5..bc4c2945 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.5.4 \ No newline at end of file + - idds-common==0.6.0 \ No newline at end of file From 56874ba61ba85c1a49a82edb7c085bc2e81bb7a2 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 17:29:57 +0200 Subject: [PATCH 023/156] fix task definition --- .../lib/idds/atlas/workflow/atlaspandawork.py | 28 +++++++++---------- main/lib/idds/tests/test_atlaspandawork.py | 18 ++++++------ 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index 94440524..2f42a64c 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -143,21 +143,19 @@ def parse_task_parameters(self, task_parameters): if 'jobParameters' in self.task_parameters: jobParameters = self.task_parameters['jobParameters'] - for jobPs in jobParameters: - if type(jobPs) in [tuple, list]: - for jobP in jobPs: - if type(jobP) in [dict]: - if 'dataset' in jobP and 'param_type' in jobP: - if jobP['param_type'] == 'input': - input_c = jobP['dataset'] - scope, name = extract_scope_atlas(input_c, scopes=[]) - input_coll = {'scope': scope, 'name': name} - self.set_primary_input_collection(input_coll) - if jobP['param_type'] == 'output': - output_c = jobP['dataset'] - scope, name = extract_scope_atlas(output_c, scopes=[]) - output_coll = {'scope': scope, 'name': name} - self.add_output_collections([output_coll]) + for jobP in jobParameters: + if type(jobP) in [dict]: + if 'dataset' in jobP and 'param_type' in jobP: + if jobP['param_type'] == 'input': + input_c = jobP['dataset'] + scope, name = extract_scope_atlas(input_c, scopes=[]) + input_coll = {'scope': scope, 'name': name} + self.set_primary_input_collection(input_coll) + if jobP['param_type'] == 'output': + output_c = jobP['dataset'] + scope, name = extract_scope_atlas(output_c, scopes=[]) + output_coll = {'scope': scope, 'name': name} + self.add_output_collections([output_coll]) if 'log' in self.task_parameters: log = self.task_parameters['log'] diff --git a/main/lib/idds/tests/test_atlaspandawork.py b/main/lib/idds/tests/test_atlaspandawork.py index 10f5b109..743b2040 100644 --- a/main/lib/idds/tests/test_atlaspandawork.py +++ b/main/lib/idds/tests/test_atlaspandawork.py @@ -53,16 +53,14 @@ def get_workflow(): "type": "constant", "value": "-a jobO.6f1dc5a8-eeb3-4cdf-aed0-0930b6a0e815.tar.gz" }, - [ - { - "container": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", - "dataset": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", - "hidden": True, - "param_type": "output", - "type": "template", - "value": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt" - } - ], + { + "container": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", + "dataset": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", + "hidden": True, + "param_type": "output", + "type": "template", + "value": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt" + }, { "type": "constant", "value": "-o \"{'seed.txt': 'user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt'}\"" From b08952560d29ce1f0f1d4ac1195027cf09383bc6 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 17:42:23 +0200 Subject: [PATCH 024/156] disable stomp timeout traceback error --- atlas/lib/idds/atlas/notifier/messaging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atlas/lib/idds/atlas/notifier/messaging.py b/atlas/lib/idds/atlas/notifier/messaging.py index e55433cb..b895b73b 100644 --- a/atlas/lib/idds/atlas/notifier/messaging.py +++ b/atlas/lib/idds/atlas/notifier/messaging.py @@ -21,7 +21,9 @@ from idds.common.plugin.plugin_base import PluginBase from idds.common.utils import setup_logging + setup_logging(__name__) +logging.getLogger("stomp").setLevel(logging.CRITICAL) class MessagingListener(stomp.ConnectionListener): From 2d840b2cf22564b432bf3c07f338d6b99ea2edcb Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 17:42:40 +0200 Subject: [PATCH 025/156] fix atlas panda work --- atlas/lib/idds/atlas/workflow/atlaspandawork.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index 2f42a64c..de8f8b86 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -600,9 +600,6 @@ def poll_processing_updates(self, processing, input_output_maps): proc.tofinish = False proc.toforcefinish = False proc.polling_retries = 0 - elif self.is_all_contents_terminated_and_with_missing(input_output_maps): - self.logger.info("All contents terminated(There are Missing contents). Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) - self.kill_processing(processing) processing_status, poll_updated_contents, new_input_output_maps = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) From b4215f39cfdf2c272878ff6d907469d04ade64b8 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 4 Aug 2021 18:02:21 +0200 Subject: [PATCH 026/156] fix workload_id in processing monitor --- main/lib/idds/orm/requests.py | 3 +++ main/lib/idds/rest/v1/monitor.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/main/lib/idds/orm/requests.py b/main/lib/idds/orm/requests.py index 8850d7fa..72b3f074 100644 --- a/main/lib/idds/orm/requests.py +++ b/main/lib/idds/orm/requests.py @@ -271,6 +271,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta elif with_processing: subquery = session.query(models.Processing.processing_id, models.Processing.transform_id, + models.Processing.workload_id, models.Processing.status.label("processing_status"), models.Processing.created_at.label("processing_created_at"), models.Processing.updated_at.label("processing_updated_at"), @@ -301,6 +302,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Transform.workload_id.label("transform_workload_id"), models.Transform.status.label("transform_status"), subquery.c.processing_id, + subquery.c.workload_id.label("processing_workload_id"), subquery.c.processing_status, subquery.c.processing_created_at, subquery.c.processing_updated_at, @@ -327,6 +329,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Transform.workload_id.label("transform_workload_id"), models.Transform.status.label("transform_status"), subquery.c.processing_id, + subquery.c.workload_id.label("processing_workload_id"), subquery.c.processing_status, subquery.c.processing_created_at, subquery.c.processing_updated_at, diff --git a/main/lib/idds/rest/v1/monitor.py b/main/lib/idds/rest/v1/monitor.py index 89ace7e1..c66ab642 100644 --- a/main/lib/idds/rest/v1/monitor.py +++ b/main/lib/idds/rest/v1/monitor.py @@ -110,7 +110,7 @@ def get_requests(self, request_id, workload_id, with_request=False, with_transfo reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=False, with_processing=True, with_metadata=False) for req in reqs: ret = {'request_id': req['request_id'], - 'workload_id': req['workload_id'], + 'workload_id': req['processing_workload_id'], 'processing_id': req['processing_id'], 'processing_status': req['processing_status'].name if req['processing_status'] else req['processing_status'], 'processing_created_at': req['processing_created_at'], From ef5729debd9fd51a5522db898e92e8893ae1e771 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 5 Aug 2021 17:15:08 +0200 Subject: [PATCH 027/156] add started status for work --- .../lib/idds/atlas/workflow/atlaspandawork.py | 15 ++++++- common/lib/idds/common/constants.py | 1 + doma/lib/idds/doma/workflow/domapandawork.py | 13 +++++- main/lib/idds/agents/clerk/clerk.py | 9 ++-- workflow/lib/idds/workflow/work.py | 42 +++++++++++++++++++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index de8f8b86..48a7fb82 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -702,9 +702,20 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True if self.is_processings_finished(): self.status = WorkStatus.Finished - if self.is_processings_failed(): - self.status = WorkStatus.Failed elif self.is_processings_subfinished(): self.status = WorkStatus.SubFinished + elif self.is_processings_failed(): + self.status = WorkStatus.Failed + elif self.is_processings_expired(): + self.status = WorkStatus.Expired + elif self.is_processings_cancelled(): + self.status = WorkStatus.Cancelled + elif self.is_processings_suspended(): + self.status = WorkStatus.Suspended + elif self.is_processings_running(): + self.status = WorkStatus.Running else: self.status = WorkStatus.Transforming + + if self.is_processings_started(): + self.started = True diff --git a/common/lib/idds/common/constants.py b/common/lib/idds/common/constants.py index 1db78204..4b8a6a98 100644 --- a/common/lib/idds/common/constants.py +++ b/common/lib/idds/common/constants.py @@ -111,6 +111,7 @@ class WorkStatus(IDDSEnum): ToExpire = 15 Expiring = 16 Expired = 17 + Running = 18 class RequestStatus(IDDSEnum): diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index b68bfdaa..32d2b321 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -367,10 +367,12 @@ def create_processing(self, input_output_maps=[]): task_param_map['transUses'] = '' task_param_map['transHome'] = None if self.encode_command_line: - task_param_map['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c-enc' + # task_param_map['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c-enc' + task_param_map['transPath'] = 'https://storage.googleapis.com/drp-us-central1-containers/bash-c-enc' task_param_map['encJobParams'] = True else: - task_param_map['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c' + # task_param_map['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c' + task_param_map['transPath'] = 'https://storage.googleapis.com/drp-us-central1-containers/bash-c' task_param_map['processingType'] = self.processingType task_param_map['prodSourceLabel'] = self.prodSourceLabel task_param_map['taskType'] = self.task_type @@ -937,3 +939,10 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True self.status = WorkStatus.Failed else: self.status = WorkStatus.SubFinished + elif self.is_processings_running(): + self.status = WorkStatus.Running + else: + self.status = WorkStatus.Transforming + + if self.is_processings_started(): + self.started = True diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index 80c811fc..8e5ea235 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -284,10 +284,11 @@ def process_running_request_real(self, req): for work in works: # print(work.get_work_id()) tf = core_transforms.get_transform(transform_id=work.get_work_id()) - transform_work = tf['transform_metadata']['work'] - # work_status = WorkStatus(tf['status'].value) - # work.set_status(work_status) - work.sync_work_data(status=tf['status'], substatus=tf['substatus'], work=transform_work) + if tf: + transform_work = tf['transform_metadata']['work'] + # work_status = WorkStatus(tf['status'].value) + # work.set_status(work_status) + work.sync_work_data(status=tf['status'], substatus=tf['substatus'], work=transform_work) wf.refresh_works() is_operation = False diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index f0a48b7a..3697b839 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -434,6 +434,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.release_inputs_after_submitting = release_inputs_after_submitting self.has_new_inputs = True + self.started = False self.status = WorkStatus.New self.substatus = WorkStatus.New self.polling_retries = 0 @@ -576,6 +577,14 @@ def has_new_inputs(self): def has_new_inputs(self, value): self.add_metadata_item('has_new_inputs', value) + @property + def started(self): + return self.get_metadata_item('started', False) + + @started.setter + def started(self, value): + self.add_metadata_item('started', value) + @property def status(self): return self.get_metadata_item('status', WorkStatus.New) @@ -944,6 +953,14 @@ def get_backup_to_release_inputs(self): self.backup_to_release_inputs['0'] = [] return to_release_inputs + def is_started(self): + return self.started + + def is_running(self): + if self.status in [WorkStatus.Running]: + return True + return False + def is_terminated(self): """ *** Function called by Transformer agent. @@ -1400,6 +1417,26 @@ def reap_processing(self, processing): else: self.logger.error("Cannot reap an unterminated processing: %s" % processing) + def is_processings_started(self): + """ + *** Function called by Transformer agent. + """ + for p_id in self.active_processings: + p = self.processings[p_id] + if p.submitted_at: + return True + return False + + def is_processings_running(self): + """ + *** Function called by Transformer agent. + """ + for p_id in self.active_processings: + p = self.processings[p_id] + if p.status in [ProcessingStatus.Running]: + return True + return False + def is_processings_terminated(self): """ *** Function called by Transformer agent. @@ -1633,8 +1670,13 @@ def syn_work_status(self, input_output_maps, all_updates_flushed=True, output_st self.status = WorkStatus.Cancelled elif self.is_processings_suspended(): self.status = WorkStatus.Suspended + elif self.is_processings_running(): + self.status = WorkStatus.Running else: self.status = WorkStatus.Transforming + + if self.is_processings_started(): + self.started = True self.logger.debug("syn_work_status(%s): work.status: %s" % (str(self.get_processing_ids()), str(self.status))) def sync_work_data(self, status, substatus, work): From 93c63d56485b0ea1fc1c7b1c5cc0c55dd098c0cd Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 5 Aug 2021 17:15:57 +0200 Subject: [PATCH 028/156] add composite condi --- workflow/lib/idds/workflow/workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index 89b2cd71..ab827733 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -102,10 +102,10 @@ class ConditionOperator(Enum): Or = 1 -class MultiCondition(Condition): +class CompositeCondition(Condition): def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): - super(MultiCondition, self).__init__(cond=cond, current_work=current_work, true_work=true_work, - false_work=false_work, logger=logger) + super(CompositeCondition, self).__init__(cond=cond, current_work=current_work, true_work=true_work, + false_work=false_work, logger=logger) self.additional_conds = [] def add_condition(self, cond, current_work, operator=ConditionOperator.And): From 39e4eee7382f1bee081cd6c15156ad8858f8093b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 5 Aug 2021 17:16:12 +0200 Subject: [PATCH 029/156] update test --- main/lib/idds/tests/core_tests.py | 2 +- main/lib/idds/tests/test_migrate_requests.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index b4b97d4a..93edebbf 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,7 +102,7 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=132, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=148, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index b161c92f..d23947fc 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -29,14 +29,14 @@ def migrate(): # atlas atlas_host = 'https://aipanda181.cern.ch:443/idds' # noqa F841 - cm1 = ClientManager(host=doma_host) + cm1 = ClientManager(host=dev_host) # reqs = cm1.get_requests(request_id=290) old_request_id = 72533 - for old_request_id in [93]: + for old_request_id in [148]: # for old_request_id in [60]: # noqa E115 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=doma_host) + cm2 = ClientManager(host=dev_host) for req in reqs: req = convert_old_req_2_workflow_req(req) workflow = req['request_metadata']['workflow'] From 13399bdbc05eaaab93391a1184c07919c3db234f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 5 Aug 2021 17:16:49 +0200 Subject: [PATCH 030/156] new version 0.6.1 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index aeed08aa..c4233d3d 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 8cbd4d20..60ea13b9 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.6.0 - - idds-workflow==0.6.0 \ No newline at end of file + - idds-common==0.6.1 + - idds-workflow==0.6.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index aeed08aa..c4233d3d 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 241fe2e4..e022a418 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.6.0 - - idds-workflow==0.6.0 \ No newline at end of file + - idds-common==0.6.1 + - idds-workflow==0.6.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index aeed08aa..c4233d3d 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 7532bfb8..aa82c621 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 86ca910a..b43e7ecc 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.6.0 - - idds-workflow==0.6.0 \ No newline at end of file + - idds-common==0.6.1 + - idds-workflow==0.6.1 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index aeed08aa..c4233d3d 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index e62ae9d4..1eac76d3 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.6.0 - - idds-workflow==0.6.0 - - idds-client==0.6.0 \ No newline at end of file + - idds-common==0.6.1 + - idds-workflow==0.6.1 + - idds-client==0.6.1 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index aeed08aa..c4233d3d 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/website/version.py b/website/version.py index aeed08aa..c4233d3d 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index aeed08aa..c4233d3d 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.0" +release_version = "0.6.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index bc4c2945..1719383f 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.6.0 \ No newline at end of file + - idds-common==0.6.1 \ No newline at end of file From 8a624ae32c4cf1fb82cfedc83c7a7bd5c9f19e70 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 10 Aug 2021 14:11:59 +0200 Subject: [PATCH 031/156] improve index on contents table --- main/lib/idds/orm/contents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/lib/idds/orm/contents.py b/main/lib/idds/orm/contents.py index 1f581257..ac464506 100644 --- a/main/lib/idds/orm/contents.py +++ b/main/lib/idds/orm/contents.py @@ -347,7 +347,7 @@ def get_contents_by_transform(transform_id, to_json=False, session=None): try: query = session.query(models.Content) - query = query.with_hint(models.Content, "INDEX(CONTENTS CONTENTS_REQ_TF_COLL_IDX)", 'oracle') + query = query.with_hint(models.Content, "INDEX(CONTENTS CONTENT_ID_UQ)", 'oracle') query = query.filter(models.Content.transform_id == transform_id) query = query.order_by(asc(models.Content.map_id)) @@ -423,7 +423,7 @@ def get_content_status_statistics(coll_id=None, session=None): """ try: query = session.query(models.Content.status, func.count(models.Content.content_id)) - query = query.with_hint(models.Content, "INDEX(CONTENTS CONTENTS_REQ_TF_COLL_IDX)", 'oracle') + query = query.with_hint(models.Content, "INDEX(CONTENTS CONTENTS_ID_NAME_IDX)", 'oracle') if coll_id: query = query.filter(models.Content.coll_id == coll_id) query = query.group_by(models.Content.status) From 8390ebae8d25bd1a7aa4411c88200bbf256228f2 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 10 Aug 2021 14:24:48 +0200 Subject: [PATCH 032/156] add index for messages table --- main/etc/sql/oracle_11.sql | 3 +++ main/etc/sql/oracle_19.sql | 2 ++ main/lib/idds/orm/messages.py | 1 + 3 files changed, 6 insertions(+) diff --git a/main/etc/sql/oracle_11.sql b/main/etc/sql/oracle_11.sql index 66855c69..93ee285a 100644 --- a/main/etc/sql/oracle_11.sql +++ b/main/etc/sql/oracle_11.sql @@ -417,6 +417,9 @@ CREATE OR REPLACE TRIGGER TRIG_MESSAGE_ID alter table messages add destination NUMBER(2); alter table messages add processing_id NUMBER(12); +CREATE INDEX MESSAGES_TYPE_ST_IDX ON MESSAGES (msg_type, status, destination, request_id); + + --- health CREATE SEQUENCE HEALTH_ID_SEQ MINVALUE 1 INCREMENT BY 1 START WITH 1 NOCACHE NOORDER NOCYCLE; CREATE TABLE HEALTH diff --git a/main/etc/sql/oracle_19.sql b/main/etc/sql/oracle_19.sql index f8da3d7b..346f103d 100644 --- a/main/etc/sql/oracle_19.sql +++ b/main/etc/sql/oracle_19.sql @@ -323,6 +323,8 @@ CREATE TABLE MESSAGES alter table messages add destination NUMBER(2); alter table messages add processing_id NUMBER(12); +CREATE INDEX MESSAGES_TYPE_ST_IDX ON MESSAGES (msg_type, status, destination, request_id); + --- health CREATE SEQUENCE HEALTH_ID_SEQ MINVALUE 1 INCREMENT BY 1 START WITH 1 NOCACHE ORDER NOCYCLE GLOBAL; CREATE TABLE HEALTH diff --git a/main/lib/idds/orm/messages.py b/main/lib/idds/orm/messages.py index 01e6d53f..ba8f46c1 100644 --- a/main/lib/idds/orm/messages.py +++ b/main/lib/idds/orm/messages.py @@ -126,6 +126,7 @@ def retrieve_messages(bulk_size=1000, msg_type=None, status=None, source=None, messages = [] try: query = session.query(models.Message) + query = query.with_hint(models.Message, "INDEX(MESSAGES MESSAGES_TYPE_ST_IDX)", 'oracle') if msg_type is not None: query = query.filter_by(msg_type=msg_type) if status is not None: From 1ccee7cde25fbca01688e175d6b24102cb72815d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 10 Aug 2021 15:44:15 +0200 Subject: [PATCH 033/156] add message index --- main/lib/idds/orm/base/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/lib/idds/orm/base/models.py b/main/lib/idds/orm/base/models.py index 48b7fe24..fc790bff 100644 --- a/main/lib/idds/orm/base/models.py +++ b/main/lib/idds/orm/base/models.py @@ -545,7 +545,8 @@ class Message(BASE, ModelBase): updated_at = Column("updated_at", DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) msg_content = Column(JSON()) - _table_args = (PrimaryKeyConstraint('msg_id', name='MESSAGES_PK')) + _table_args = (PrimaryKeyConstraint('msg_id', name='MESSAGES_PK'), + Index('MESSAGES_TYPE_ST_IDX', 'msg_type', 'status', 'destination', 'request_id')) def register_models(engine): From 99fe41785dbf16e5b301e05ea3d6e1c985acabe3 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:31:58 +0200 Subject: [PATCH 034/156] work base to support logger --- workflow/lib/idds/workflow/base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/workflow/lib/idds/workflow/base.py b/workflow/lib/idds/workflow/base.py index a0aa0185..24fd2198 100644 --- a/workflow/lib/idds/workflow/base.py +++ b/workflow/lib/idds/workflow/base.py @@ -6,8 +6,9 @@ # http://www.apache.org/licenses/LICENSE-2.0OA # # Authors: -# - Wen Guan, , 2020 +# - Wen Guan, , 2020 - 2021 +import logging import pickle import urllib # from enum import Enum @@ -74,3 +75,12 @@ def serialize(self): def deserialize(obj): # return urllib.parse.unquote_to_bytes(pickle.loads(obj)) return pickle.loads(urllib.parse.unquote_to_bytes(obj)) + + def get_class_name(self): + return self.__class__.__name__ + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) From badf2bb4bde1584510ab03d05ef9fd9788784fbc Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:32:44 +0200 Subject: [PATCH 035/156] work to support non template --- workflow/lib/idds/workflow/work.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index 3697b839..72806ea1 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -348,11 +348,14 @@ def processing(self, value): self.status = self._processing.get('status', None) self.substatus = self._processing.get('substatus', None) self.processing_metadata = self._processing.get('processing_metadata', None) + self.submitted_at = self._processing.get('submitted_at', None) if self.processing_metadata and 'processing' in self.processing_metadata: proc = self.processing_metadata['processing'] self.work = proc.work self.external_id = proc.external_id self.errors = proc.errors + if not self.submitted_at: + self.submitted_at = proc.submitted_at self.output_data = self._processing.get('output_metadata', None) @@ -366,7 +369,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, work_tag=None, exec_type='local', sandbox=None, work_id=None, work_name=None, primary_input_collection=None, other_input_collections=None, output_collections=None, log_collections=None, release_inputs_after_submitting=False, - agent_attributes=None, + agent_attributes=None, is_template=False, logger=None): """ Init a work/task/transformation. @@ -388,6 +391,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.internal_id = str(uuid.uuid1()) self.template_work_id = self.internal_id + self.is_template = is_template self.class_name = self.__class__.__name__.lower() self.initialized = False self.sequence_id = 0 @@ -806,6 +810,9 @@ def set_work_name(self, work_name): def get_work_name(self): return self.work_name + def get_is_template(self): + self.is_template + def setup_logger(self): """ Setup logger @@ -1079,7 +1086,8 @@ def generate_work_from_template(self): self.logger = logger new_work.logger = logger # new_work.template_work_id = self.get_internal_id() - new_work.internal_id = str(uuid.uuid1()) + if self.is_template: + new_work.internal_id = str(uuid.uuid1()) return new_work def get_template_id(self): @@ -1421,7 +1429,8 @@ def is_processings_started(self): """ *** Function called by Transformer agent. """ - for p_id in self.active_processings: + # for p_id in self.active_processings: + for p_id in self.processings: p = self.processings[p_id] if p.submitted_at: return True @@ -1675,7 +1684,7 @@ def syn_work_status(self, input_output_maps, all_updates_flushed=True, output_st else: self.status = WorkStatus.Transforming - if self.is_processings_started(): + if self.is_processings_terminated() or self.is_processings_running() or self.is_processings_started(): self.started = True self.logger.debug("syn_work_status(%s): work.status: %s" % (str(self.get_processing_ids()), str(self.status))) From ab84590a3004acff7c7fc47b26599298437eccd7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:34:03 +0200 Subject: [PATCH 036/156] fix to parse input coll, to parse task id from reuse --- .../lib/idds/atlas/workflow/atlaspandawork.py | 116 ++++++++++-------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index 48a7fb82..f7314390 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -62,6 +62,7 @@ def __init__(self, task_parameters=None, self.panda_url_ssl = None self.panda_monitor = None + self.task_type = 'test' self.task_parameters = None self.parse_task_parameters(task_parameters) # self.logger.setLevel(logging.DEBUG) @@ -141,6 +142,9 @@ def parse_task_parameters(self, task_parameters): self.task_name = self.task_parameters['taskName'] self.set_work_name(self.task_name) + if 'prodSourceLabel' in self.task_parameters: + self.task_type = self.task_parameters['prodSourceLabel'] + if 'jobParameters' in self.task_parameters: jobParameters = self.task_parameters['jobParameters'] for jobP in jobParameters: @@ -191,55 +195,58 @@ def poll_external_collection(self, coll): if coll.status in [CollectionStatus.Closed]: return coll else: - if coll.coll_type == CollectionType.PseudoDataset: - coll.coll_metadata['bytes'] = 0 - coll.coll_metadata['total_files'] = 0 - coll.coll_metadata['availability'] = True - coll.coll_metadata['events'] = 0 - coll.coll_metadata['is_open'] = False - coll.coll_metadata['run_number'] = None - coll.coll_metadata['did_type'] = 'DATASET' - coll.coll_metadata['list_all_files'] = False - - if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: - coll_status = CollectionStatus.Closed - else: - coll_status = CollectionStatus.Open - coll.status = coll_status - - coll.coll_metadata['coll_type'] = coll.coll_type - - return coll + try: + if not coll.coll_type == CollectionType.PseudoDataset: + client = self.get_rucio_client() + did_meta = client.get_metadata(scope=coll.scope, name=coll.name) + + coll.coll_metadata['bytes'] = did_meta['bytes'] + coll.coll_metadata['total_files'] = did_meta['length'] + coll.coll_metadata['availability'] = did_meta['availability'] + coll.coll_metadata['events'] = did_meta['events'] + coll.coll_metadata['is_open'] = did_meta['is_open'] + coll.coll_metadata['run_number'] = did_meta['run_number'] + coll.coll_metadata['did_type'] = did_meta['did_type'] + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + if 'did_type' in coll.coll_metadata: + if coll.coll_metadata['did_type'] == 'DATASET': + coll_type = CollectionType.Dataset + elif coll.coll_metadata['did_type'] == 'CONTAINER': + coll_type = CollectionType.Container + else: + coll_type = CollectionType.File + else: + coll_type = CollectionType.Dataset + coll.coll_metadata['coll_type'] = coll_type + coll.coll_type = coll_type + return coll + except Exception as ex: + self.logger.warn("Faield to get the dataset information(%s:%s) from Panda: %s" % (coll.scope, coll.name, str(ex))) + + coll.coll_metadata['bytes'] = 0 + coll.coll_metadata['total_files'] = 0 + coll.coll_metadata['availability'] = True + coll.coll_metadata['events'] = 0 + coll.coll_metadata['is_open'] = False + coll.coll_metadata['run_number'] = None + coll.coll_metadata['did_type'] = 'DATASET' + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed else: - client = self.get_rucio_client() - did_meta = client.get_metadata(scope=coll.scope, name=coll.name) - - coll.coll_metadata['bytes'] = did_meta['bytes'] - coll.coll_metadata['total_files'] = did_meta['length'] - coll.coll_metadata['availability'] = did_meta['availability'] - coll.coll_metadata['events'] = did_meta['events'] - coll.coll_metadata['is_open'] = did_meta['is_open'] - coll.coll_metadata['run_number'] = did_meta['run_number'] - coll.coll_metadata['did_type'] = did_meta['did_type'] - coll.coll_metadata['list_all_files'] = False - - if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: - coll_status = CollectionStatus.Closed - else: - coll_status = CollectionStatus.Open - coll.status = coll_status + coll_status = CollectionStatus.Open + coll.status = coll_status + + coll.coll_metadata['coll_type'] = coll.coll_type - if 'did_type' in coll.coll_metadata: - if coll.coll_metadata['did_type'] == 'DATASET': - coll_type = CollectionType.Dataset - elif coll.coll_metadata['did_type'] == 'CONTAINER': - coll_type = CollectionType.Container - else: - coll_type = CollectionType.File - else: - coll_type = CollectionType.Dataset - coll.coll_metadata['coll_type'] = coll_type - coll.coll_type = coll_type return coll except Exception as ex: self.logger.error(ex) @@ -391,7 +398,18 @@ def submit_panda_task(self, processing): task_param = proc.processing_metadata['task_param'] return_code = Client.insertTaskParams(task_param, verbose=True) if return_code[0] == 0: - return return_code[1][1] + try: + task_id = int(return_code[1][1]) + return task_id + except Exception as ex: + self.logger.warn("task id is not retruned: (%s) is not task id: %s" % (return_code[1][1], str(ex))) + # jediTaskID=26468582 + if return_code[1][1] and 'jediTaskID=' in return_code[1][1]: + parts = return_code[1][1].split(" ") + for part in parts: + if 'jediTaskID=' in part: + task_id = int(part.split("=")[1]) + return task_id else: self.logger.warn("submit_panda_task, return_code: %s" % str(return_code)) except Exception as ex: @@ -717,5 +735,5 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True else: self.status = WorkStatus.Transforming - if self.is_processings_started(): + if self.is_processings_terminated() or self.is_processings_running() or self.is_processings_started(): self.started = True From bb1d549f8de95b398beb88e6f18b31c162f33fc5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:35:12 +0200 Subject: [PATCH 037/156] helper to release inputs --- common/lib/idds/common/utils.py | 8 +++++++- main/lib/idds/agents/clerk/clerk.py | 26 +++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index 732ef5f1..cabaa4a9 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -6,7 +6,7 @@ # http://www.apache.org/licenses/LICENSE-2.0OA # # Authors: -# - Wen Guan, , 2019 - 2020 +# - Wen Guan, , 2019 - 2021 import datetime @@ -471,3 +471,9 @@ def extract_scope_atlas(did, scopes): if did.endswith('/'): did = did[:-1] return scope, did + + +def truncate_string(string, length=800): + string = (string[:length] + '...') if string and len(string) > length else string + return string + diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index 8e5ea235..520da4f6 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -20,10 +20,12 @@ from idds.common import exceptions from idds.common.constants import (Sections, RequestStatus, RequestLocking, TransformStatus, MessageType, MessageStatus, - MessageSource, MessageDestination) -from idds.common.utils import setup_logging + MessageSource, MessageDestination, + ContentStatus, ContentRelationType) +from idds.common.utils import setup_logging, truncate_string from idds.core import (requests as core_requests, - transforms as core_transforms) + transforms as core_transforms, + catalog as core_catalog) from idds.agents.common.baseagent import BaseAgent setup_logging(__name__) @@ -151,7 +153,7 @@ def process_new_request(self, req): ret_req = {'request_id': req['request_id'], 'parameters': {'status': RequestStatus.Failed, 'locking': RequestLocking.Idle, - 'errors': {'msg': '%s: %s' % (ex, traceback.format_exc())}}} + 'errors': {'msg': truncate_string('%s: %s' % (ex, traceback.format_exc()), length=900)}}} return ret_req def process_new_requests(self): @@ -334,7 +336,7 @@ def process_running_request_real(self, req): 'locking': RequestLocking.Idle, 'next_poll_at': next_poll_at, 'request_metadata': req['request_metadata'], - 'errors': {'msg': req_msg}} + 'errors': {'msg': truncate_string(req_msg, 900)}} new_messages = [] if req_status == RequestStatus.ToExpire: @@ -378,11 +380,25 @@ def process_running_request_message(self, req, messages): 'errors': {'msg': '%s: %s' % (ex, traceback.format_exc())}}} return ret_req + def release_inputs(self, request_id): + contents = core_catalog.get_contents(request_id=request_id, status=ContentStatus.Available) + ret_contents = {} + for content in contents: + if content['content_relation_type'] == ContentRelationType.Output: # InputDependency + if content['coll_id'] not in ret_contents: + ret_contents[content['coll_id']] = [] + ret_contents[content['coll_id']].append(content) + + updated_contents = core_transforms.release_inputs_by_collection(ret_contents) + + core_catalog.update_contents(updated_contents) + def process_running_request(self, req): """ process running request """ try: + self.release_inputs(req['request_id']) ret_req = self.process_running_request_real(req) except Exception as ex: self.logger.error(ex) From db038c9e37bc48e25d2ac24fe9a861e60b36ddd1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:36:03 +0200 Subject: [PATCH 038/156] remove session of read_session at the end --- main/lib/idds/orm/base/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/lib/idds/orm/base/session.py b/main/lib/idds/orm/base/session.py index c4013884..03a610c4 100644 --- a/main/lib/idds/orm/base/session.py +++ b/main/lib/idds/orm/base/session.py @@ -253,7 +253,7 @@ def new_funct(*args, **kwargs): try: kwargs['session'] = session result = function(*args, **kwargs) - session.close() + session.remove() return result except TimeoutError as error: session.rollback() # pylint: disable=maybe-no-member From e964257f1c07be9b3e8b4bebadbeee52137c405d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:37:37 +0200 Subject: [PATCH 039/156] update tests --- main/lib/idds/tests/core_tests.py | 2 +- main/lib/idds/tests/test_atlaspandawork.py | 222 +++++++++++++------ main/lib/idds/tests/test_migrate_requests.py | 2 +- main/lib/idds/tests/test_workflow.py | 64 ++---- main/lib/idds/tests/trigger_release.py | 4 +- 5 files changed, 174 insertions(+), 120 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 93edebbf..73f4a02c 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,7 +102,7 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=148, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=161, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/test_atlaspandawork.py b/main/lib/idds/tests/test_atlaspandawork.py index 743b2040..1b5b33e9 100644 --- a/main/lib/idds/tests/test_atlaspandawork.py +++ b/main/lib/idds/tests/test_atlaspandawork.py @@ -19,82 +19,162 @@ def get_workflow(): - from idds.workflow.workflow import Workflow + from idds.workflow.workflow import Workflow, Condition from idds.atlas.workflow.atlaspandawork import ATLASPandaWork - task_parameters = {"architecture": "", - "cliParams": "prun --exec 'echo %RNDM:10 > seed.txt' --outputs seed.txt --nJobs 3 --outDS user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top", - "excludedSite": [], - "includedSite": None, - "jobParameters": [ - { - "type": "constant", - "value": "-j \"\" --sourceURL ${SURL}" - }, - { - "type": "constant", - "value": "-r ." - }, - { - "padding": False, - "type": "constant", - "value": "-p \"" - }, - { - "padding": False, - "type": "constant", - "value": "echo%20%25RNDM%3A10%20%3E%20seed.txt" - }, - { - "type": "constant", - "value": "\"" - }, - { - "type": "constant", - "value": "-a jobO.6f1dc5a8-eeb3-4cdf-aed0-0930b6a0e815.tar.gz" - }, - { - "container": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", - "dataset": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top_seed.txt/", - "hidden": True, - "param_type": "output", - "type": "template", - "value": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt" - }, - { - "type": "constant", - "value": "-o \"{'seed.txt': 'user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.$JEDITASKID._${SN/P}.seed.txt'}\"" - } - ], - "log": { - "container": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top/", - "dataset": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top/", - "param_type": "log", - "type": "template", - "value": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top.log.$JEDITASKID.${SN}.log.tgz" - }, - "nEvents": 3, - "nEventsPerJob": 1, - "nMaxFilesPerJob": 200, - "noInput": True, - "osInfo": "Linux-3.10.0-1160.31.1.el7.x86_64-x86_64-with-centos-7.9.2009-Core", - "processingType": "panda-client-1.4.79-jedi-run", - "prodSourceLabel": "user", - "respectSplitRule": True, - "site": None, - "sourceURL": "https://aipanda059.cern.ch:25443", - "taskName": "user.tmaeno.34ba141d-242c-43cf-a326-7730a5dc7338_000_top", - "transHome": None, - "transUses": "", - "uniqueTaskName": True, - "userName": "Tadashi Maeno", - "vo": "atlas" - } + task_parameters1 = {"architecture": "", + "cliParams": "prun --exec 'echo %RNDM:10 > seed.txt' --outputs seed.txt --nJobs 2 --outDS user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top", + "excludedSite": [], + "includedSite": None, + "jobParameters": [ + { + "type": "constant", + "value": "-j \"\" --sourceURL ${SURL}" + }, + { + "type": "constant", + "value": "-r ." + }, + { + "padding": False, + "type": "constant", + "value": "-p \"" + }, + { + "padding": False, + "type": "constant", + "value": "echo%20%25RNDM%3A10%20%3E%20seed.txt" + }, + { + "type": "constant", + "value": "\"" + }, + { + "type": "constant", + "value": "-a jobO.185663cd-6df9-4ac8-adf9-0d9bf9d5892e.tar.gz" + }, + { + "container": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top_seed.txt/", + "dataset": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top_seed.txt/", + "hidden": True, + "param_type": "output", + "type": "template", + "value": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top.$JEDITASKID._${SN/P}.seed.txt" + }, + { + "type": "constant", + "value": "-o \"{'seed.txt': 'user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top.$JEDITASKID._${SN/P}.seed.txt'}\"" + } + ], + "log": { + "container": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top/", + "dataset": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top/", + "param_type": "log", + "type": "template", + "value": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top.log.$JEDITASKID.${SN}.log.tgz" + }, + "nEvents": 2, + "nEventsPerJob": 1, + "nMaxFilesPerJob": 200, + "noInput": True, + "osInfo": "Linux-3.10.0-1160.36.2.el7.x86_64-x86_64-with-centos-7.9.2009-Core", + "processingType": "panda-client-1.4.80-jedi-run", + "prodSourceLabel": "user", + "respectSplitRule": True, + "site": None, + "sourceURL": "https://aipanda048.cern.ch:25443", + "taskName": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top", + "transHome": None, + "transUses": "", + "uniqueTaskName": True, + "userName": "Tadashi Maeno", + "vo": "atlas"} - work = ATLASPandaWork(task_parameters=task_parameters) + task_parameters2 = {"architecture": "", + "cliParams": "prun --exec 'echo %IN > results.root' --outputs results.root --forceStaged --inDS user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top_seed.txt/ --outDS user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom", + "excludedSite": [], + "includedSite": None, + "jobParameters": [ + { + "type": "constant", + "value": "-j \"\" --sourceURL ${SURL}" + }, + { + "type": "constant", + "value": "-r ." + }, + { + "padding": False, + "type": "constant", + "value": "-p \"" + }, + { + "padding": False, + "type": "constant", + "value": "echo%20%25IN%20%3E%20results.root" + }, + { + "type": "constant", + "value": "\"" + }, + { + "type": "constant", + "value": "-a jobO.185663cd-6df9-4ac8-adf9-0d9bf9d5892e.tar.gz" + }, + { + "dataset": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top_seed.txt/", + "exclude": "\\.log\\.tgz(\\.\\d+)*$", + "expand": True, + "param_type": "input", + "type": "template", + "value": "-i \"${IN/T}\"" + }, + { + "container": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom_results.root/", + "dataset": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom_results.root/", + "hidden": True, + "param_type": "output", + "type": "template", + "value": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom.$JEDITASKID._${SN/P}.results.root" + }, + { + "type": "constant", + "value": "-o \"{'results.root': 'user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom.$JEDITASKID._${SN/P}.results.root'}\"" + } + ], + "log": { + "container": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom/", + "dataset": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom/", + "param_type": "log", + "type": "template", + "value": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom.log.$JEDITASKID.${SN}.log.tgz" + }, + "nMaxFilesPerJob": 200, + "noWaitParent": True, + "osInfo": "Linux-3.10.0-1160.36.2.el7.x86_64-x86_64-with-centos-7.9.2009-Core", + "parentTaskName": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top", + "processingType": "panda-client-1.4.80-jedi-run", + "prodSourceLabel": "user", + "respectSplitRule": True, + "site": None, + "sourceURL": "https://aipanda048.cern.ch:25443", + "taskName": "user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_001_bottom", + "transHome": None, + "transUses": "", + "uniqueTaskName": True, + "userName": "Tadashi Maeno", + "vo": "atlas"} + + work1 = ATLASPandaWork(task_parameters=task_parameters1) + work2 = ATLASPandaWork(task_parameters=task_parameters2) wf = Workflow() wf.set_workload_id(234567) - wf.add_work(work) + wf.add_work(work1) + wf.add_work(work2) + + # cond = Condition(cond=work1.is_finished, true_work=work2) + cond = Condition(cond=work1.is_started, true_work=work2) + wf.add_condition(cond) return wf diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index d23947fc..813e6061 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -32,7 +32,7 @@ def migrate(): cm1 = ClientManager(host=dev_host) # reqs = cm1.get_requests(request_id=290) old_request_id = 72533 - for old_request_id in [148]: + for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) diff --git a/main/lib/idds/tests/test_workflow.py b/main/lib/idds/tests/test_workflow.py index 043377f9..e4b52fdf 100644 --- a/main/lib/idds/tests/test_workflow.py +++ b/main/lib/idds/tests/test_workflow.py @@ -6,7 +6,7 @@ # http://www.apache.org/licenses/LICENSE-2.0OA # # Authors: -# - Wen Guan, , 2020 +# - Wen Guan, , 2020 - 2021 """ @@ -21,7 +21,7 @@ from idds.common.constants import RequestType, RequestStatus from idds.common.utils import get_rest_host -from idds.workflow.work import Work, Parameter, WorkStatus +from idds.workflow.work import Work # Parameter, WorkStatus from idds.workflow.workflow import Condition, Workflow @@ -36,30 +36,22 @@ def init(self): work2 = Work(executable='echo', arguments='--in=IN_DATASET --out=OUT_DATASET', sandbox=None, - parameters=Parameter({'IN_DATASET': 'data17:data17.test.raw.1', - 'OUT_DATASET': 'data17:data17.test.work2'}), work_id=2, - input_collection_scope='data17', - input_collection_name='data17.test.raw.1', - output_collection_scope='data17', - output_collection_name='data17.test.work2') + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) work3 = Work(executable='echo', arguments='--in=IN_DATASET --out=OUT_DATASET', sandbox=None, - parameters=Parameter({'IN_DATASET': 'data17:data17.test.work2', - 'OUT_DATASET': 'data17:data17.test.work3'}), work_id=3, - input_collection_scope='data17', - input_collection_name='data17.test.work2', - output_collection_scope='data17', - output_collection_name='data17.test.work3') + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) workflow = Workflow() workflow.add_work(work1, initial=True) workflow.add_work(work2, initial=True) workflow.add_work(work3, initial=False) - cond = Condition(cond=work2.is_finished, prework=work2, true_work=work3) + cond = Condition(cond=work2.is_finished, true_work=work3) workflow.add_condition(cond) return workflow @@ -71,52 +63,34 @@ def test_workflow(self): work2 = Work(executable='echo', arguments='--in=IN_DATASET --out=OUT_DATASET', sandbox=None, - parameters=Parameter({'IN_DATASET': 'data17:data17.test.raw.1', - 'OUT_DATASET': 'data17:data17.test.work2'}), work_id=2, - input_collection_scope='data17', - input_collection_name='data17.test.raw.1', - output_collection_scope='data17', - output_collection_name='data17.test.work2') + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) work3 = Work(executable='echo', arguments='--in=IN_DATASET --out=OUT_DATASET', sandbox=None, - parameters=Parameter({'IN_DATASET': 'data17:data17.test.work2', - 'OUT_DATASET': 'data17:data17.test.work3'}), work_id=3, - input_collection_scope='data17', - input_collection_name='data17.test.work2', - output_collection_scope='data17', - output_collection_name='data17.test.work3') + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) workflow = Workflow() workflow.add_work(work1, initial=True) workflow.add_work(work2, initial=True) workflow.add_work(work3, initial=False) - cond = Condition(cond=work2.is_finished, prework=work2, true_work=work3) + cond = Condition(cond=work2.is_finished, true_work=work3) + # print(cond.all_works()) workflow.add_condition(cond) # check - workflow_str = workflow.serialize() - workflow1 = Workflow.deserialize(workflow_str) + # workflow_str = workflow.serialize() + # workflow1 = Workflow.deserialize(workflow_str) # print(workflow_str) # print(workflow1) - works = workflow1.get_current_works() - # print([str(work) for work in works]) - assert(works == [work1, work2]) - - works = workflow.get_current_works() - # print(works) - assert(works == [work1, work2]) - - workflow.update_work_status(work1, WorkStatus.Finished) works = workflow.get_current_works() - assert(works == [work2]) - - workflow.update_work_status(work2, WorkStatus.Finished) - works = workflow.get_current_works() - assert(works == [work3]) + # print([str(work) for work in works]) + # print([w.work_id for w in works]) + assert(works == []) def test_workflow_request(self): workflow = self.init() @@ -130,7 +104,7 @@ def test_workflow_request(self): 'status': RequestStatus.New, 'priority': 0, 'lifetime': 30, - 'request_metadata': {'workload_id': '20776840', 'workflow': workflow.serialize()} + 'request_metadata': {'workload_id': '20776840', 'workflow': workflow} } # print(props) diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 168fc24d..91573e77 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -11,7 +11,7 @@ from idds.orm.contents import get_input_contents # noqa F401 -contents = get_contents(request_id=107, status=ContentStatus.Available) +contents = get_contents(request_id=202, status=ContentStatus.Available) ret_contents = {} for content in contents: if content['content_relation_type'] == ContentRelationType.Output: # InputDependency @@ -26,6 +26,6 @@ updated_contents = core_transforms.release_inputs_by_collection(ret_contents) for update_content in updated_contents: print(update_content) - break + # break update_contents(updated_contents) From c6f968946034c2e70663bfe26595994856537e8c Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:38:10 +0200 Subject: [PATCH 040/156] add condition tests --- .../lib/idds/tests/test_workflow_condition.py | 641 ++++++++++++++++++ 1 file changed, 641 insertions(+) create mode 100644 main/lib/idds/tests/test_workflow_condition.py diff --git a/main/lib/idds/tests/test_workflow_condition.py b/main/lib/idds/tests/test_workflow_condition.py new file mode 100644 index 00000000..dd708a97 --- /dev/null +++ b/main/lib/idds/tests/test_workflow_condition.py @@ -0,0 +1,641 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 + + +""" +Test workflow condtions. +""" + +import unittest2 as unittest +# from nose.tools import assert_equal +from idds.common.utils import setup_logging + +from idds.common.utils import json_dumps, json_loads + +from idds.workflow.work import Work, WorkStatus +from idds.workflow.workflow import CompositeCondition, AndCondition, OrCondition, Condition, ConditionTrigger, Workflow + + +setup_logging(__name__) + + +class TestWorkflowCondtion(unittest.TestCase): + + def test_condition(self): + # init_p = Parameter({'input_dataset': 'data17:data17.test.raw.1'}) + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) + work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) + work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) + work7 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=7, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) + work8 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=8, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) + + workflow = Workflow() + workflow.add_work(work1, initial=True) + workflow.add_work(work2, initial=True) + workflow.add_work(work3, initial=False) + workflow.add_work(work8, initial=False) + + # CompositeCondition + cond1 = CompositeCondition(conditions=work1.is_finished, true_works=work2, false_works=work3) + works = cond1.all_works() + assert(works == [work1, work2, work3]) + works = cond1.all_pre_works() + assert(works == [work1]) + works = cond1.all_next_works() + assert(works == [work2, work3]) + cond_status = cond1.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond1.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + + works = cond1.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond1.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work3]) + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work2]) + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + + works = cond1.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond1.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + # CompositeCondition + cond2 = CompositeCondition(conditions=[work1.is_finished, work2.is_finished, work3.is_finished], true_works=[work4, work5], false_works=[work6, work7]) + + works = cond2.all_works() + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond2.all_pre_works() + assert(works == [work1, work2, work3]) + works = cond2.all_next_works() + assert(works == [work4, work5, work6, work7]) + cond_status = cond2.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond2.get_condition_status() + assert(cond_status is False) + work2.status = WorkStatus.Finished + cond_status = cond2.get_condition_status() + assert(cond_status is False) + work3.status = WorkStatus.Finished + cond_status = cond2.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond2.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond2.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work6, work7]) + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work4, work5]) + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond2.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond2.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + # AndCondition + cond3 = AndCondition(conditions=[work1.is_finished, work2.is_finished, work3.is_finished], true_works=[work4, work5], false_works=[work6, work7]) + + works = cond3.all_works() + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond3.all_pre_works() + assert(works == [work1, work2, work3]) + works = cond3.all_next_works() + assert(works == [work4, work5, work6, work7]) + cond_status = cond3.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond3.get_condition_status() + assert(cond_status is False) + work2.status = WorkStatus.Finished + cond_status = cond3.get_condition_status() + assert(cond_status is False) + work3.status = WorkStatus.Finished + cond_status = cond3.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond3.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond3.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work6, work7]) + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work4, work5]) + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond3.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond3.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + # OrCondtion + cond4 = OrCondition(conditions=[work1.is_finished, work2.is_finished, work3.is_finished], true_works=[work4, work5], false_works=[work6, work7]) + + works = cond4.all_works() + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond4.all_pre_works() + assert(works == [work1, work2, work3]) + works = cond4.all_next_works() + assert(works == [work4, work5, work6, work7]) + cond_status = cond4.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond4.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.Finished + cond_status = cond4.get_condition_status() + assert(cond_status is True) + work2.status = WorkStatus.New + work3.status = WorkStatus.Finished + cond_status = cond4.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond4.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + # work2.status = WorkStatus.Finished + # work3.status = WorkStatus.Finished + works = cond4.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work6, work7]) + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + # work2.status = WorkStatus.Finished + # work3.status = WorkStatus.Finished + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work4, work5]) + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond4.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + # work2.status = WorkStatus.Finished + # work3.status = WorkStatus.Finished + works = cond4.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + # Condition + cond5 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + + works = cond5.all_works() + assert(works == [work1, work2, work3]) + works = cond5.all_pre_works() + assert(works == [work1]) + works = cond5.all_next_works() + assert(works == [work2, work3]) + cond_status = cond5.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond5.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + + works = cond5.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond5.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work3]) + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work2]) + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + + works = cond5.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond5.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + # multiple conditions + cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + + works = cond7.all_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond7.all_pre_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4, work5]) + works = cond7.all_next_works() + works.sort(key=lambda x: x.work_id) + # print([w.work_id for w in works]) + assert(works == [work2, work3, work6, work7]) + cond_status = cond7.get_condition_status() + assert(cond_status is False) + + work4.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is False) + work5.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is True) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + return workflow + + def print_workflow(self, workflow): + print('print workflow') + print(workflow.conditions) + for cond_id in workflow.conditions: + print(cond_id) + cond = workflow.conditions[cond_id] + print(cond) + print(cond.conditions) + print(cond.true_works) + print(cond.false_works) + for w in cond.true_works: + print(w) + if isinstance(w, CompositeCondition): + print(w.conditions) + print(w.true_works) + print(w.false_works) + + def test_workflow(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) + work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) + work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) + work7 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=7, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) + work8 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=8, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + workflow.add_work(work3, initial=False) + workflow.add_work(work4, initial=False) + workflow.add_work(work5, initial=False) + workflow.add_work(work6, initial=False) + workflow.add_work(work7, initial=False) + workflow.add_work(work8, initial=False) + + # multiple conditions + cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + + workflow.add_condition(cond7) + id_works = workflow.independent_works + # print(id_works) + id_works.sort() + id_works_1 = [work1, work4, work5, work8] + id_works_1 = [w.get_template_id() for w in id_works_1] + id_works_1.sort() + # id_works.sort(key=lambda x: x.work_id) + assert(id_works == id_works_1) + + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow1 = json_loads(workflow_str) + # print('before load_metadata') + # self.print_workflow(workflow1) + workflow1.load_metadata() + # print('after load_metadata') + # self.print_workflow(workflow1) + workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4) + assert(workflow_str == workflow_str1) + + works = cond7.all_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond7.all_pre_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4, work5]) + works = cond7.all_next_works() + works.sort(key=lambda x: x.work_id) + # print([w.work_id for w in works]) + assert(works == [work2, work3, work6, work7]) + cond_status = cond7.get_condition_status() + assert(cond_status is False) + + work4.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is False) + work5.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is True) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + return workflow + + def test_workflow_condition_reload(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) + work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) + work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) + work7 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=7, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) + work8 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=8, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + workflow.add_work(work3, initial=False) + workflow.add_work(work4, initial=False) + workflow.add_work(work5, initial=False) + workflow.add_work(work6, initial=False) + workflow.add_work(work7, initial=False) + workflow.add_work(work8, initial=False) + + # multiple conditions + cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + + workflow.add_condition(cond7) + + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow1 = json_loads(workflow_str) + # print('before load_metadata') + # self.print_workflow(workflow1) + workflow1.load_metadata() + # print('after load_metadata') + # self.print_workflow(workflow1) + workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4) + assert(workflow_str == workflow_str1) + + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + return workflow From 5a471e8a7c582c581cdea7d9897fdbae7aa6954d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 20 Aug 2021 20:39:52 +0200 Subject: [PATCH 041/156] add condition docs --- docs/source/index.rst | 1 + docs/source/users/condition_examples.rst | 62 ++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 docs/source/users/condition_examples.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 3dd0a1e9..10de37b5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -55,6 +55,7 @@ User Documentation users/installing_client users/cli_examples + users/condition_examples users/admin_guides users/contributing diff --git a/docs/source/users/condition_examples.rst b/docs/source/users/condition_examples.rst new file mode 100644 index 00000000..b6eb20c5 --- /dev/null +++ b/docs/source/users/condition_examples.rst @@ -0,0 +1,62 @@ +Conditions: Examples +============================= + +iDDS provides composite conditions to support complicated workflows. + +conditions +~~~~~~~~~~~~~~~~~~~~~~~~ + +1. condtions + +Here are conditions iDDS supports + +.. code-block:: python + + from idds.workflow.work import Work, WorkStatus + from idds.workflow.workflow import (CompositeCondition, AndCondition, OrCondition, Condition, + ConditionOperator, ConditionTrigger, Workflow) + + com_cond = CompositeCondition(operator=ConditionOperator.And, conditions=[], true_works=[], false_works=[]) + + and_cond = AndCondition(conditions=[], true_works=[], false_works=[]) + = CompositeCondition(operator=ConditionOperator.And, conditions=[], true_works=[], false_works=[]) + + or_cond = OrCondition(conditions=[], true_works=[], false_works=[]) + = CompositeCondition(operator=ConditionOperator.Or, conditions=[], true_works=[], false_works=[]) + + # To support old conditions + cond = Condition(cond=, true_work=, false_work=) + = CompositeCondition(operator=ConditionOperator.And, conditions=[cond], true_works=[true_work], false_works=[false_work]) + + + # combine multiple condtions into one condition + and_cond = AndCondition(conditions=[work5.is_terminated], true_works=[work6]) + or_cond = OrCondition(conditions=[work7.is_terminated, work8.is_terminated], true_works=[work9]) + conds = AndCondition(conditions=[work1.is_finished, work2.is_started], + true_works=[work3, or_cond], false_works=[work4, and_cond]) + # for combined conditions, only need to add the top(top tree) condition to the workflow. + # Since every add_condition will create an evaluation entrypoint in iDDS. + # If a branch of a combined condition is added to a workflow with add_condition, this branch will be evaluated as a separate condition tree. + workflow.add_condition(conds) + + +2. condition trigger + +Condition trigger is an option for iDDS to process whether to remember whether a condition work is already triggered. It's used to avoid duplicated triggering some processes(For example, when using work1.is_started to trigger work2. If the condition is not recorded, work2 can be triggered many times every time when the condition is evaluated). + +.. code-block:: python + + class ConditionTrigger(IDDSEnum): + NotTriggered = 0 + ToTrigger = 1 + Triggered = 2 + + # ToTrigger will return untriggered works based on the conditions and mark the work as triggered. + # exception: if the work.is_template is true, even the condition work is marked as triggered, the work will still be triggered. So for cases such as work.is_started should not be used as a condition for works with is_template=True. + cond.get_next_works(trigger=ConditionTrigger.ToTrigger) + + # Will only return untriggered works based on conditions. It will not update the trigger status. + cond.get_next_works(trigger=ConditionTrigger.NotTriggered) + + # Will only return triggered works based on conditions. It will not update the trigger status. + cond.get_next_works(trigger=ConditionTrigger.Triggered) From 69c1901e5dcb5b0c70f8680105d1f6cb6e3f01d5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 21 Aug 2021 00:19:17 +0200 Subject: [PATCH 042/156] fix common utils --- common/lib/idds/common/utils.py | 1 - monitor/conf.js | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index cabaa4a9..c89d873a 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -476,4 +476,3 @@ def extract_scope_atlas(did, scopes): def truncate_string(string, length=800): string = (string[:length] + '...') if string and len(string) > length else string return string - diff --git a/monitor/conf.js b/monitor/conf.js index a212aad7..d28b96be 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus765.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus765.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus765.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus765.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus765.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus765.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus760.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus760.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus760.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus760.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus760.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus760.cern.ch:443/idds/monitor/null/null/false/false/true" } From ee48c3f24ef68461ef432f5c1560466f1f4e651e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 21 Aug 2021 00:20:18 +0200 Subject: [PATCH 043/156] new composite condition --- workflow/lib/idds/workflow/workflow.py | 585 +++++++++++++++++++++---- 1 file changed, 500 insertions(+), 85 deletions(-) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index ab827733..98d1a576 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -16,70 +16,361 @@ import time import uuid -from enum import Enum +from idds.common import exceptions +from idds.common.constants import IDDSEnum from idds.common.utils import json_dumps, setup_logging, get_proxy from idds.common.utils import str_to_date from .base import Base +from .work import Work setup_logging(__name__) -class Condition(Base): - def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): - """ - Condition. - if cond() is true, return true_work, else return false_work. +class ConditionOperator(IDDSEnum): + And = 0 + Or = 1 - :param cond: executable return true or false. - :param work: The current Work instance. - :param true_work: Work instance. - :param false_work: Work instance. - """ - if cond and callable(cond): + +class ConditionTrigger(IDDSEnum): + NotTriggered = 0 + ToTrigger = 1 + Triggered = 2 + + +class CompositeCondition(Base): + def __init__(self, operator=ConditionOperator.And, conditions=[], true_works=None, false_works=None, logger=None): + self._conditions = [] + self._true_works = [] + self._false_works = [] + + super(CompositeCondition, self).__init__() + + self.internal_id = str(uuid.uuid1()) + self.template_id = self.internal_id + + self.logger = logger + if self.logger is None: + self.setup_logger() + + if conditions is None: + conditions = [] + if true_works is None: + true_works = [] + if false_works is None: + false_works = [] + if conditions and type(conditions) not in [tuple, list]: + conditions = [conditions] + if true_works and type(true_works) not in [tuple, list]: + true_works = [true_works] + if false_works and type(false_works) not in [tuple, list]: + false_works = [false_works] + self.validate_conditions(conditions) + + self.operator = operator + self.conditions = [] + self.true_works = [] + self.false_works = [] + + self.conditions = conditions + self.true_works = true_works + self.false_works = false_works + + # real_conditions = [] + # for cond in conditions: + # # real_conditions.append({'condition': cond, 'current_work': cond.__self__}) + # real_conditions.append(cond) + # self.conditions = real_conditions + + # real_true_works, real_false_works = [], [] + # for true_work in true_works: + # # real_true_works.append({'work': true_work, 'triggered': False}) + # real_true_works.append(true_work) + # self.true_works = real_true_works + # for false_work in false_works: + # # real_false_works.append({'work': false_work, 'triggered': False}) + # real_false_works.append(false_work) + # self.false_works = real_false_works + + # self.set_class_metadata() + + # def set_class_metadata(self): + # self.add_metadata_item('class', {'class': self.__class__.__name__, + # 'module': self.__class__.__module__}) + # self.add_metadata_item('operator', self.operator.value) + + def get_class_name(self): + return self.__class__.__name__ + + def get_internal_id(self): + return self.internal_id + + def get_template_id(self): + return self.template_id + + @property + def conditions(self): + # return self.get_metadata_item('true_works', []) + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + new_value = [] + for cond in value: + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_class_id': cond.__self__.get_template_id()} + else: + new_cond = cond + new_value.append(new_cond) + self.add_metadata_item('conditions', new_value) + + @property + def true_works(self): + # return self.get_metadata_item('true_works', []) + return self._true_works + + @true_works.setter + def true_works(self, value): + self._true_works = value + true_work_meta = self.get_metadata_item('true_works', {}) + for work in value: + if isinstance(work, Work): + if work.get_template_id() not in true_work_meta: + true_work_meta[work.get_template_id()] = {'triggered': False} + elif isinstance(work, CompositeCondition): + if work.get_template_id() not in true_work_meta: + true_work_meta[work.get_template_id()] = {'triggered': False, + 'metadata': work.metadata} + self.add_metadata_item('true_works', true_work_meta) + + @property + def false_works(self): + # return self.get_metadata_item('false_works', []) + return self._false_works + + @false_works.setter + def false_works(self, value): + self._false_works = value + false_work_meta = self.get_metadata_item('false_works', {}) + for work in value: + if isinstance(work, Work): + if work.get_template_id() not in false_work_meta: + false_work_meta[work.get_template_id()] = {'triggered': False} + elif isinstance(work, CompositeCondition): + if work.get_template_id() not in false_work_meta: + false_work_meta[work.get_template_id()] = {'triggered': False, + 'metadata': work.metadata} + self.add_metadata_item('false_works', false_work_meta) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + for cond in conditions: assert(inspect.ismethod(cond)) - assert(cond.__self__ == current_work) - self.cond = cond - self.current_work = None - if current_work: - self.current_work = current_work.get_template_id() - self.true_work = None - if true_work: - self.true_work = true_work.get_template_id() - self.false_work = None - if false_work: - self.false_work = false_work.get_template_id() + assert(isinstance(cond.__self__, Work)) + if (cond.__self__.is_template): + raise exceptions.IDDSException("Work class for CompositeCondition must not be a template") + + def add_condition(self, cond): + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + if (cond.__self__.is_template): + raise exceptions.IDDSException("Work class for CompositeCondition must not be a template") + + # self.conditions.append({'condition': cond, 'current_work': cond.__self__}) + + self._conditions.append(cond) + new_value = self.get_metadata_item('conditions', []) + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_class_id': cond.__self__.get_template_id()} + else: + new_cond = cond + new_value.append(new_cond) + self.add_metadata_item('conditions', new_value) + + def load_metadata(self): + # conditions = self.get_metadata_item('conditions', []) + true_works_meta = self.get_metadata_item('true_works', {}) + false_works_meta = self.get_metadata_item('false_works', {}) + + for work in self.true_works: + if isinstance(work, CompositeCondition): + if work.get_template_id() in true_works_meta: + work.metadata = true_works_meta[work.get_template_id()]['metadata'] + for work in self.false_works: + if isinstance(work, CompositeCondition): + if work.get_template_id() in false_works_meta: + work.metadata = false_works_meta[work.get_template_id()]['metadata'] + + def to_dict(self): + # print('to_dict') + ret = {'class': self.__class__.__name__, + 'module': self.__class__.__module__, + 'attributes': {}} + for key, value in self.__dict__.items(): + # print(key) + # print(value) + # if not key.startswith('__') and not key.startswith('_'): + if not key.startswith('__'): + if key == 'logger': + value = None + elif key == '_conditions': + new_value = [] + for cond in value: + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_class_id': cond.__self__.get_template_id()} + else: + new_cond = cond + new_value.append(new_cond) + value = new_value + elif key in ['_true_works', '_false_works']: + new_value = [] + for w in value: + if isinstance(w, Work): + new_w = w.get_template_id() + elif isinstance(w, CompositeCondition): + new_w = w.to_dict() + else: + new_w = w + new_value.append(new_w) + value = new_value + else: + value = self.to_dict_l(value) + ret['attributes'][key] = value + return ret + + def get_work_from_id(self, work_id, works, works_template): + for w_id in works: + if works[w_id].get_template_id() == work_id: + return works[w_id] + for w_id in works_template: + if works_template[w_id].get_template_id() == work_id: + return works_template[w_id] + return None + + def load_conditions(self, works, works_template): + new_conditions = [] + for cond in self.conditions: + if 'idds_method' in cond and 'idds_method_class_id' in cond: + class_id = cond['idds_method_class_id'] + work = self.get_work_from_id(class_id, works, works_template) + if work is not None: + new_cond = getattr(work, cond['idds_method']) + else: + self.logger.error("Work cannot be found for %s" % class_id) + new_cond = cond + else: + new_cond = cond + new_conditions.append(new_cond) + self.conditions = new_conditions + + new_true_works = [] + for w in self.true_works: + if isinstance(w, CompositeCondition): + # work = w.load_conditions(works, works_template) + w.load_conditions(works, works_template) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, works, works_template) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_true_works.append(work) + self.true_works = new_true_works + + new_false_works = [] + for w in self.false_works: + if isinstance(w, CompositeCondition): + # work = w.load_condtions(works, works_template) + w.load_conditions(works, works_template) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, works, works_template) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_false_works.append(work) + self.false_works = new_false_works def all_works(self): works = [] - works.append(self.current_work) - if self.true_work: - works.append(self.true_work) - if self.false_work: - works.append(self.false_work) + works = works + self.all_pre_works() + works = works + self.all_next_works() + return works + + def all_condition_ids(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__.get_template_id()) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_condition_ids() + return works + + def all_pre_works(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_pre_works() return works def all_next_works(self): works = [] - if self.true_work: - works.append(self.true_work) - if self.false_work: - works.append(self.false_work) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_next_works() + else: + works.append(work) return works - def get_cond_status(self): - if callable(self.cond): - if self.cond(): + def get_current_cond_status(self, cond): + if callable(cond): + if cond(): return True else: return False else: - if self.cond: + if cond: return True else: return False + def get_cond_status(self): + if self.operator == ConditionOperator.And: + for cond in self.conditions: + if not self.get_current_cond_status(cond): + return False + return True + else: + for cond in self.conditions: + if self.get_current_cond_status(cond): + return True + return False + + def get_condition_status(self): + return self.get_cond_status() + def is_condition_true(self): if self.get_cond_status(): return True @@ -90,39 +381,106 @@ def is_condition_false(self): return True return False - def get_next_work(self): + def get_next_works(self, trigger=ConditionTrigger.NotTriggered): + works = [] if self.get_cond_status(): - return self.true_work + true_work_meta = self.get_metadata_item('true_works', {}) + for work in self.true_works: + if isinstance(work, CompositeCondition): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_template_id() not in true_work_meta: + true_work_meta[work.get_template_id()] = {'triggered': False} + if trigger == ConditionTrigger.ToTrigger: + if not true_work_meta[work.get_template_id()]['triggered']: + true_work_meta[work.get_template_id()]['triggered'] = True + works.append(work) + elif work.get_is_template(): + # A template can be triggered many times. + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not true_work_meta[work.get_template_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if true_work_meta[work.get_template_id()]['triggered']: + works.append(work) + self.add_metadata_item('true_works', true_work_meta) else: - return self.false_work + false_work_meta = self.get_metadata_item('false_works', {}) + for work in self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_template_id() not in false_work_meta: + false_work_meta[work.get_template_id()] = {'triggered': False} + if trigger == ConditionTrigger.ToTrigger: + if not false_work_meta[work.get_template_id()]['triggered']: + false_work_meta[work.get_template_id()]['triggered'] = True + works.append(work) + elif work.get_is_template(): + # A template can be triggered many times. + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not false_work_meta[work.get_template_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if false_work_meta[work.get_template_id()]['triggered']: + works.append(work) + self.add_metadata_item('false_works', false_work_meta) + return works -class ConditionOperator(Enum): - And = 0 - Or = 1 +class AndCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(AndCondition, self).__init__(operator=ConditionOperator.And, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class OrCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(OrCondition, self).__init__(operator=ConditionOperator.Or, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) -class CompositeCondition(Condition): +class Condition(CompositeCondition): def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): - super(CompositeCondition, self).__init__(cond=cond, current_work=current_work, true_work=true_work, - false_work=false_work, logger=logger) - self.additional_conds = [] + super(Condition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) - def add_condition(self, cond, current_work, operator=ConditionOperator.And): - cond = {'cond': cond, 'current_work': current_work.get_template_id(), 'operator': operator} - self.additional_conds.append(cond) - def get_cond_status(self): - if callable(self.cond): - if self.cond(): - return True - else: - return False - else: - if self.cond: - return True - else: - return False +class TemplateCondition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + if true_work is not None and not isinstance(true_work, Work): + raise exceptions.IDDSException("true_work can only be set with Work class") + if false_work is not None and not isinstance(false_work, Work): + raise exceptions.IDDSException("false_work can only be set with Work class") + + super(TemplateCondition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + if len(conditions) > 1: + raise exceptions.IDDSException("Condition class can only support one condition. To support multiple condition, please use CompositeCondition.") + for cond in conditions: + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + + def add_condition(self, cond): + raise exceptions.IDDSException("Condition class doesn't support add_condition. To support multiple condition, please use CompositeCondition.") class Workflow(Base): @@ -131,6 +489,8 @@ def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None """ Init a workflow. """ + self._conditions = {} + self._work_conds = {} super(Workflow, self).__init__() self.internal_id = str(uuid.uuid1()) @@ -161,7 +521,6 @@ def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None # if the primary initial_work is not set, it's the first initial work. self.primary_initial_work = None self.independent_works = [] - self.work_conds = {} self.first_initial = False self.new_to_run_works = [] @@ -324,6 +683,35 @@ def load_works(self): if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): self.last_updated_at = work.last_updated_at + @property + def conditions(self): + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + conditions_metadata = {} + if self._conditions: + for k in self._conditions: + conditions_metadata[k] = self._conditions[k].metadata + self.add_metadata_item('conditions', conditions_metadata) + + @property + def work_conds(self): + return self._work_conds + + @work_conds.setter + def work_conds(self, value): + self._work_conds = value + # self.add_metadata_item('work_conds', value) + + def load_work_conditions(self): + conditions_metadata = self.get_metadata_item('conditions', {}) + for cond_id in self.conditions: + self.conditions[cond_id].load_conditions(self.works, self.get_works_template()) + if cond_id in conditions_metadata: + self.conditions[cond_id].metadata = conditions_metadata[cond_id] + @property def work_sequence(self): return self.get_metadata_item('work_sequence', {}) @@ -438,6 +826,7 @@ def to_update_transforms(self, value): def load_metadata(self): self.load_works() + self.load_work_conditions() def get_class_name(self): return self.__class__.__name__ @@ -534,7 +923,7 @@ def add_work(self, work, initial=False, primary=False): if initial: if primary: self.primary_initial_work = work.get_template_id() - self.add_initial_works(work.get_template_id()) + self.add_initial_works(work) self.independent_works.append(work.get_template_id()) @@ -542,18 +931,29 @@ def add_condition(self, cond): self.first_initial = False cond_works = cond.all_works() for cond_work in cond_works: - assert(cond_work in self.get_works_template()) - - if cond.current_work not in self.work_conds: - self.work_conds[cond.current_work] = [] - self.work_conds[cond.current_work].append(cond) + assert(cond_work.get_template_id() in self.get_works_template()) + + if cond.get_template_id() not in self.conditions: + conditions = self.conditions + conditions[cond.get_template_id()] = cond + self.conditions = conditions + + # if cond.current_work not in self.work_conds: + # self.work_conds[cond.current_work] = [] + # self.work_conds[cond.current_work].append(cond) + work_conds = self.work_conds + for work in cond.all_pre_works(): + if work.get_template_id() not in work_conds: + work_conds[work.get_template_id()] = [] + work_conds[work.get_template_id()].append(cond.get_template_id()) + self.work_conds = work_conds # if a work is a true_work or false_work of a condition, # should remove it from independent_works cond_next_works = cond.all_next_works() for next_work in cond_next_works: - if next_work in self.independent_works: - self.independent_works.remove(next_work) + if next_work.get_template_id() in self.independent_works: + self.independent_works.remove(next_work.get_template_id()) def add_initial_works(self, work): assert(work.get_template_id() in self.get_works_template()) @@ -561,21 +961,27 @@ def add_initial_works(self, work): if self.primary_initial_work is None: self.primary_initial_work = work.get_template_id() - def enable_next_work(self, work, cond): + def enable_next_works(self, work, cond): self.log_debug("Checking Work %s condition: %s" % (work.get_internal_id(), json_dumps(cond, sort_keys=True, indent=4))) - if cond and self.is_class_method(cond.cond): - # cond_work_id = self.works[cond.cond['idds_method_class_id']] - cond.cond = getattr(work, cond.cond['idds_method']) - self.log_info("Work %s condition: %s" % (work.get_internal_id(), cond.cond)) - next_work = cond.get_next_work() + # load_conditions should cover it. + # if cond and self.is_class_method(cond.cond): + # # cond_work_id = self.works[cond.cond['idds_method_class_id']] + # cond.cond = getattr(work, cond.cond['idds_method']) + + self.log_info("Work %s condition: %s" % (work.get_internal_id(), cond.conditions)) + next_works = cond.get_next_works(trigger=ConditionTrigger.ToTrigger) self.log_info("Work %s condition status %s" % (work.get_internal_id(), cond.get_cond_status())) - self.log_info("Work %s next work %s" % (work.get_internal_id(), next_work)) - if next_work is not None: - new_parameters = work.get_parameters_for_next_task() - next_work = self.get_new_work_from_template(next_work, new_parameters) - work.add_next_work(next_work.get_internal_id()) - return next_work + self.log_info("Work %s next works %s" % (work.get_internal_id(), str(next_works))) + new_next_works = [] + if next_works is not None: + for next_work in next_works: + new_parameters = work.get_parameters_for_next_task() + new_next_work = self.get_new_work_from_template(next_work.get_template_id(), new_parameters) + work.add_next_work(new_next_work.get_internal_id()) + # cond.add_condition_work(new_next_work) ####### TODO: + new_next_works.append(new_next_work) + return new_next_works def __str__(self): return str(json_dumps(self)) @@ -690,6 +1096,15 @@ def sync_works(self): self.current_running_works.append(work.get_internal_id()) for work in [self.works[k] for k in self.current_running_works]: + if work.get_template_id() in self.work_conds: + self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + json_dumps(self.work_conds[work.get_template_id()], sort_keys=True, indent=4))) + for cond_id in self.work_conds[work.get_template_id()]: + cond = self.conditions[cond_id] + self.log_debug("Work %s has condition dependencie %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + self.enable_next_works(work, cond) + if work.is_terminated(): self.log_info("Work %s is terminated(%s)" % (work.get_internal_id(), work.get_status())) self.log_debug("Work conditions: %s" % json_dumps(self.work_conds, sort_keys=True, indent=4)) @@ -699,10 +1114,10 @@ def sync_works(self): self.terminated_works.append(work.get_internal_id()) self.current_running_works.remove(work.get_internal_id()) else: - self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), - json_dumps(self.work_conds[work.get_template_id()], sort_keys=True, indent=4))) - for cond in self.work_conds[work.get_template_id()]: - self.enable_next_work(work, cond) + # self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + # json_dumps(self.work_conds[work.get_template_id()], sort_keys=True, indent=4))) + # for cond in self.work_conds[work.get_template_id()]: + # self.enable_next_works(work, cond) self.terminated_works.append(work.get_internal_id()) self.current_running_works.remove(work.get_internal_id()) From 49e05b90643ec75b56e795e8b21e9950476a5ee7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 21 Aug 2021 11:14:00 +0200 Subject: [PATCH 044/156] update docs --- docs/source/users/condition_examples.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/users/condition_examples.rst b/docs/source/users/condition_examples.rst index b6eb20c5..3e67c742 100644 --- a/docs/source/users/condition_examples.rst +++ b/docs/source/users/condition_examples.rst @@ -42,6 +42,7 @@ Here are conditions iDDS supports 2. condition trigger +(This part is for iDDS developers. Users normally should not use this function.) Condition trigger is an option for iDDS to process whether to remember whether a condition work is already triggered. It's used to avoid duplicated triggering some processes(For example, when using work1.is_started to trigger work2. If the condition is not recorded, work2 can be triggered many times every time when the condition is evaluated). .. code-block:: python From a17e6ec743244cf9796477b02cb8480a4de734b6 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 24 Aug 2021 14:29:31 +0200 Subject: [PATCH 045/156] fix atlaspandawork --- atlas/lib/idds/atlas/workflow/atlaspandawork.py | 2 +- main/lib/idds/tests/test_migrate_requests.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlaspandawork.py b/atlas/lib/idds/atlas/workflow/atlaspandawork.py index f7314390..621a8a76 100644 --- a/atlas/lib/idds/atlas/workflow/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflow/atlaspandawork.py @@ -508,7 +508,7 @@ def poll_panda_task(self, processing=None, input_output_maps=None): self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) if task_info[0] != 0: self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) - return ProcessingStatus.Submitting, {} + return ProcessingStatus.Submitting, [], {} task_info = task_info[1] diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index 813e6061..9dd0b4d5 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -29,11 +29,12 @@ def migrate(): # atlas atlas_host = 'https://aipanda181.cern.ch:443/idds' # noqa F841 - cm1 = ClientManager(host=dev_host) + cm1 = ClientManager(host=doma_host) # reqs = cm1.get_requests(request_id=290) old_request_id = 72533 - for old_request_id in [152]: + # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 + for old_request_id in [200]: # noqa E115 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) cm2 = ClientManager(host=dev_host) From 76730ccb28f66096d469375af338764fb3dbdb6b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 24 Aug 2021 14:53:29 +0200 Subject: [PATCH 046/156] fix Condition to be compatible with old conditions --- workflow/lib/idds/workflow/workflow.py | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index 98d1a576..a89f2597 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -119,6 +119,8 @@ def conditions(self, value): self._conditions = value new_value = [] for cond in value: + if cond is None: + continue if inspect.ismethod(cond): new_cond = {'idds_method': cond.__name__, 'idds_method_class_id': cond.__self__.get_template_id()} @@ -137,6 +139,8 @@ def true_works(self, value): self._true_works = value true_work_meta = self.get_metadata_item('true_works', {}) for work in value: + if work is None: + continue if isinstance(work, Work): if work.get_template_id() not in true_work_meta: true_work_meta[work.get_template_id()] = {'triggered': False} @@ -156,6 +160,8 @@ def false_works(self, value): self._false_works = value false_work_meta = self.get_metadata_item('false_works', {}) for work in value: + if work is None: + continue if isinstance(work, Work): if work.get_template_id() not in false_work_meta: false_work_meta[work.get_template_id()] = {'triggered': False} @@ -456,6 +462,34 @@ def __init__(self, cond=None, current_work=None, true_work=None, false_work=None false_works=[false_work] if false_work else [], logger=logger) + # to support load from old conditions + @property + def cond(self): + # return self.get_metadata_item('true_works', []) + return self.conditions[0] if len(self.conditions) >= 1 else None + + @cond.setter + def cond(self, value): + self.conditions = [value] + + @property + def true_work(self): + # return self.get_metadata_item('true_works', []) + return self.true_works if len(self.true_works) >= 1 else None + + @true_work.setter + def true_work(self, value): + self.true_works = [value] + + @property + def false_work(self): + # return self.get_metadata_item('true_works', []) + return self.false_works if len(self.false_works) >= 1 else None + + @false_work.setter + def false_work(self, value): + self.false_works = [value] + class TemplateCondition(CompositeCondition): def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): From 27860c1a4b0d604a03e70c766eaccde9631a07c7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 24 Aug 2021 20:44:06 +0200 Subject: [PATCH 047/156] new version 0.7.0 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/tests/test_migrate_requests.py | 3 ++- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/conf.js | 12 ++++++------ monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 15 files changed, 26 insertions(+), 25 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index c4233d3d..9dd023d8 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 60ea13b9..364a21e8 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.6.1 - - idds-workflow==0.6.1 \ No newline at end of file + - idds-common==0.7.0 + - idds-workflow==0.7.0 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index c4233d3d..9dd023d8 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index e022a418..d101daa2 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.6.1 - - idds-workflow==0.6.1 \ No newline at end of file + - idds-common==0.7.0 + - idds-workflow==0.7.0 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index c4233d3d..9dd023d8 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index aa82c621..49927018 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index b43e7ecc..6d777353 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.6.1 - - idds-workflow==0.6.1 \ No newline at end of file + - idds-common==0.7.0 + - idds-workflow==0.7.0 \ No newline at end of file diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index 9dd0b4d5..94da858b 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -34,7 +34,8 @@ def migrate(): old_request_id = 72533 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 - for old_request_id in [200]: # noqa E115 + # for old_request_id in [200]: # noqa E115 + for old_request_id in [183]: # noqa E115 # doma 183 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) cm2 = ClientManager(host=dev_host) diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index c4233d3d..9dd023d8 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 1eac76d3..6a6ba28e 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.6.1 - - idds-workflow==0.6.1 - - idds-client==0.6.1 \ No newline at end of file + - idds-common==0.7.0 + - idds-workflow==0.7.0 + - idds-client==0.7.0 \ No newline at end of file diff --git a/monitor/conf.js b/monitor/conf.js index d28b96be..f8739468 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus760.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus760.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus760.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus760.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus760.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus760.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/version.py b/monitor/version.py index c4233d3d..9dd023d8 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/website/version.py b/website/version.py index c4233d3d..9dd023d8 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index c4233d3d..9dd023d8 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.6.1" +release_version = "0.7.0" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 1719383f..0da2154e 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.6.1 \ No newline at end of file + - idds-common==0.7.0 \ No newline at end of file From b6c11a873aba887b80421d202bc277b12d134ab3 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 25 Aug 2021 11:21:29 +0200 Subject: [PATCH 048/156] option to control release helper --- main/lib/idds/agents/clerk/clerk.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index 520da4f6..cb2721e3 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -46,6 +46,13 @@ def __init__(self, num_threads=1, poll_time_period=10, retrieve_bulk_size=10, pe else: self.pending_time = None + if not hasattr(self, 'release_helper') or not self.release_helper: + self.release_helper = False + elif str(self.release_helper).lower() == 'true': + self.release_helper = True + else: + self.release_helper = False + self.new_task_queue = Queue() self.new_output_queue = Queue() self.running_task_queue = Queue() @@ -398,7 +405,8 @@ def process_running_request(self, req): process running request """ try: - self.release_inputs(req['request_id']) + if self.release_helper: + self.release_inputs(req['request_id']) ret_req = self.process_running_request_real(req) except Exception as ex: self.logger.error(ex) From 990618b6a6fbf117aa5063d25d52f22b3994e2e0 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 25 Aug 2021 15:12:26 +0200 Subject: [PATCH 049/156] fix carrier processing failures --- atlas/lib/idds/atlas/version.py | 2 +- .../idds/atlas/workflow/atlasstageinwork.py | 117 +++++++++--------- atlas/tools/env/environment.yml | 4 +- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 +- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 +- main/lib/idds/agents/carrier/carrier.py | 8 +- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +- monitor/conf.js | 12 +- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 16 files changed, 90 insertions(+), 83 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 9dd023d8..6fe421e5 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py index e3d5d01c..03c65960 100644 --- a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py @@ -314,66 +314,71 @@ def poll_processing(self, processing): return processing, 'notOk', {} def poll_processing_updates(self, processing, input_output_maps): - processing, rule_state, rep_status = self.poll_processing(processing) + try: + processing, rule_state, rep_status = self.poll_processing(processing) - updated_contents = [] - content_substatus = {'finished': 0, 'unfinished': 0} - for map_id in input_output_maps: - outputs = input_output_maps[map_id]['outputs'] - for content in outputs: - key = '%s:%s' % (content['scope'], content['name']) - if key in rep_status: - if content['substatus'] != rep_status[key]: - updated_content = {'content_id': content['content_id'], - 'substatus': rep_status[key]} - updated_contents.append(updated_content) - content['substatus'] = rep_status[key] - if content['substatus'] == ContentStatus.Available: - content_substatus['finished'] += 1 - else: - content_substatus['unfinished'] += 1 - - update_processing = {} - if rule_state == 'OK' and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Finished}} - elif self.toexpire: - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Expired}} - elif self.tocancel: - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Cancelled}} - elif self.tosuspend: - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Suspended}} - elif self.toresume: - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Running}} - update_processing['parameters']['expired_at'] = None - processing['expired_at'] = None - proc = processing['processing_metadata']['processing'] - proc.has_new_updates() - elif self.tofinish: - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.SubFinished}} - elif self.toforcefinish: + updated_contents = [] + content_substatus = {'finished': 0, 'unfinished': 0} for map_id in input_output_maps: outputs = input_output_maps[map_id]['outputs'] for content in outputs: - if content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable]: - updated_content = {'content_id': content['content_id'], - 'substatus': ContentStatus.FakeAvailable} - updated_contents.append(updated_content) - content['substatus'] = ContentStatus.FakeAvailable - - update_processing = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Finished}} - - if updated_contents: - proc = processing['processing_metadata']['processing'] - proc.has_new_updates() - - return update_processing, updated_contents, {} + key = '%s:%s' % (content['scope'], content['name']) + if key in rep_status: + if content['substatus'] != rep_status[key]: + updated_content = {'content_id': content['content_id'], + 'substatus': rep_status[key]} + updated_contents.append(updated_content) + content['substatus'] = rep_status[key] + if content['substatus'] == ContentStatus.Available: + content_substatus['finished'] += 1 + else: + content_substatus['unfinished'] += 1 + + update_processing = {} + if rule_state == 'OK' and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Finished}} + elif self.toexpire: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Expired}} + elif self.tocancel: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Cancelled}} + elif self.tosuspend: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Suspended}} + elif self.toresume: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Running}} + update_processing['parameters']['expired_at'] = None + processing['expired_at'] = None + proc = processing['processing_metadata']['processing'] + proc.has_new_updates() + elif self.tofinish: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.SubFinished}} + elif self.toforcefinish: + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable]: + updated_content = {'content_id': content['content_id'], + 'substatus': ContentStatus.FakeAvailable} + updated_contents.append(updated_content) + content['substatus'] = ContentStatus.FakeAvailable + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Finished}} + + if updated_contents: + proc = processing['processing_metadata']['processing'] + proc.has_new_updates() + + return update_processing, updated_contents, {} + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise ex def get_status_statistics(self, registered_input_output_maps): status_statistics = {} diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 364a21e8..8c1ea7d2 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.0 - - idds-workflow==0.7.0 \ No newline at end of file + - idds-common==0.7.1 + - idds-workflow==0.7.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 9dd023d8..6fe421e5 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index d101daa2..c030162d 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.0 - - idds-workflow==0.7.0 \ No newline at end of file + - idds-common==0.7.1 + - idds-workflow==0.7.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 9dd023d8..6fe421e5 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 49927018..e4a4be9c 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 6d777353..29606778 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.0 - - idds-workflow==0.7.0 \ No newline at end of file + - idds-common==0.7.1 + - idds-workflow==0.7.1 \ No newline at end of file diff --git a/main/lib/idds/agents/carrier/carrier.py b/main/lib/idds/agents/carrier/carrier.py index 6e926fc2..a72d606a 100644 --- a/main/lib/idds/agents/carrier/carrier.py +++ b/main/lib/idds/agents/carrier/carrier.py @@ -115,7 +115,8 @@ def process_new_processing(self, processing): self.logger.error(ex) self.logger.error(traceback.format_exc()) ret = {'processing_id': processing['processing_id'], - 'status': ProcessingStatus.Failed} + 'status': ProcessingStatus.Running, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4)} return ret def process_new_processings(self): @@ -384,8 +385,9 @@ def process_running_processing(self, processing): self.logger.error(ex) self.logger.error(traceback.format_exc()) processing_update = {'processing_id': processing['processing_id'], - 'parameters': {'status': ProcessingStatus.Failed, - 'locking': ProcessingLocking.Idle}} + 'parameters': {'status': ProcessingStatus.Running, + 'locking': ProcessingLocking.Idle, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4)}} ret = {'processing_update': processing_update, 'content_updates': []} return ret diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 9dd023d8..6fe421e5 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 6a6ba28e..b9ee9fb8 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.0 - - idds-workflow==0.7.0 - - idds-client==0.7.0 \ No newline at end of file + - idds-common==0.7.1 + - idds-workflow==0.7.1 + - idds-client==0.7.1 \ No newline at end of file diff --git a/monitor/conf.js b/monitor/conf.js index f8739468..b0e6853c 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7110.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7110.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7110.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7110.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7110.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7110.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/version.py b/monitor/version.py index 9dd023d8..6fe421e5 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/website/version.py b/website/version.py index 9dd023d8..6fe421e5 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 9dd023d8..6fe421e5 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.0" +release_version = "0.7.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 0da2154e..eafe21aa 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.0 \ No newline at end of file + - idds-common==0.7.1 \ No newline at end of file From bd0bed728801b5948c8e0fe64402ab500126dbe3 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 25 Aug 2021 17:04:46 +0200 Subject: [PATCH 050/156] new version 0.7.2 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/tests/resume_subfinished_data_carousel.sh | 3 +++ main/lib/idds/tests/run_sql.py | 4 +++- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 15 files changed, 24 insertions(+), 19 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 6fe421e5..8534115e 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 8c1ea7d2..62c0b081 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.1 - - idds-workflow==0.7.1 \ No newline at end of file + - idds-common==0.7.2 + - idds-workflow==0.7.2 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 6fe421e5..8534115e 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index c030162d..4be5b6ce 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.1 - - idds-workflow==0.7.1 \ No newline at end of file + - idds-common==0.7.2 + - idds-workflow==0.7.2 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 6fe421e5..8534115e 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index e4a4be9c..2470e47f 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 29606778..659912ec 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.1 - - idds-workflow==0.7.1 \ No newline at end of file + - idds-common==0.7.2 + - idds-workflow==0.7.2 \ No newline at end of file diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh index b3db9802..e921db6b 100644 --- a/main/lib/idds/tests/resume_subfinished_data_carousel.sh +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -4,6 +4,9 @@ request_ids=(75039 75065 75569 75683 76087 76181 76211 76455 76727 77281 77283 7 request_ids=(75039 77281 77283 77285 77287 83271 83483 77279 77367 77847 77947 78123 78205 80547 80827 82153) +request_ids=(75039 77281 77283 77285 77287 78533 79075 79587 80923 82301 82731 83069 83119 83271 83483 83879 83887 83891 83893 83897 83899 83905 83907 83909 83915 83921 83925 83927 83933 83939 83943 83945 83949 83955 83959 83961 83963 83967 83977 83979 83981 83983 83985 83993 83995 84067 84081 84083 84091 84095 84109 84115 84121 84125 84133 84135 84145 84147 84153 84161 84163 84167 84169 84179 84181 84185 84187 84191 84195 84199 84275 84319 84325 84399 84569 85597 86831 87541 88187 88695 89583 73 102 103 104 105 106 706 707 708 710 711 712 713 714 715 716 718 719 720 722 723 725 726 727 728 1783 1785 1787 2039 2041 2045 2047 2059 2077 2093 2119 2127 2131 2149 2155 2167 2225 2231 2233 2237 2263 2265 2267 2279 7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 16539 16541 16543 16545 16547 16549 16825 17615 19289 19319 19321 19397 19569 19583 19711 20017 20019 20115 20117 20199 20201 20203 20207 20243 21207 21487 21497 21499 21647 21673 21675 21677 21689 21699 21709 21711 21715 21717 21719 21721 21723 21725 21727 21731 21743 21761 21773 21777 21919 21923 21925 21929 21931 22037 22185 22187 22189 22193 22195 22197 22199 22201 22223 22225 22227 22285 22287 22305 22307 22309 22351 22353 22355 22357 22359 22449 22451 22453 22455 22457 22459 22461 22463 22465 22467 22469 22471 24757 24759 24761 24763 24765 24767 24769 24771 24873 24875 24877 24879 24881 24883 24885 25423 25425 25427 25429 25431 25433 25435 25437 25439 25441 25443 25445 25447 25449 25451 25453 25455 25457 25459 25461 27171 27173 27183 27185 27187 27189 27191 27193 27195 27197 27199 27201 27203 27205 27207 27209 27211 27213 27215 27217 27219 27221 27641 27643 27645 27647 27649 27651 27653 27655 27657 27659 27661 27663 27665 27667 27669 27671 27673 27675 27677 27679 27975 27977 27979 27981 27983 27985 27987 27989 27991 27993 27995 27997 27999 28001 28003 28005 28007 28009 28011 28013 29343 29345 29347 29349 29351 29353 29355 29357 29359 29361 29363 29365 29367 29369 29371 29373 29375 29377 29379 29381 29385 29639 29641 29643 29645 29647 29649 29651 29653 29655 29657 29659 29661 29663 29665 29667 29669 29671 29673 29675 29677 31559 31561 31963 31965 31967 32091 32117 32119 32121 32123 32163 32187 32189 32193 32195 32197 32207 32319 32327 32335 32337 32341 32347 32367 32541 32585 32587 36715 36717 36727 36835 36837 36839 36841 36843 36845 36847 36849 36851 36853 36855 36857 36859 36861 36863 36865 36867 36869 36871 36873 37155 37157 37159 37161 37163 37165 37167 37169 37171 37173 37175 37177 37179 37181 37183 37185 37187 37191 37193 37195 43907 43909 43911 77279 77367 77847 77947 78123 78205 80547 80827 82153 87115 87137 87199 87661 87715) + +request_ids=(73 102 103 104 105 106 706 707 708 710 711 712 713 714 715 716 718 719 720 722 723 725 726 727 728 1783 1785 1787 2039 2041 2045 2047 2059 2077 2093 2119 2127 2131 2149 2155 2167 2225 2231 2233 2237 2263 2265 2267 2279 7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 16539 16541 16543 16545 16547 16549 16825 17615 19289 19319 19321 19397 75039 77279 77281 77283 77285 77287 77367 77847 77947 78123 78205 78533 79075 79587 80109 80547 80827 80923 82153 82301 82731 83069 83119 83271 83483 83879 83887 83891 83893 83897 83915 83921 83925 83927 83933 83939 83943 83945 83949 83955 83959 83961 83963 83967 83977 83979 83981 83985 83993 83995 84067 84081 84083 84091 84095 84109 84115 84121 84125 84133 84135 84145 84147 84153 84161 84163 84167 84169 84179 84181 84185 84187 84191 84195 84199 84275 84319 84325 84335 84399 84569 85597 86831 87115 87137 87199 87541 87661 87715 87881 87993 88037 88187 88283 88353 88361 88695 88839 88841 88847 88849 88893 88991 89073 89415 89579 89583 89885 89887 89891 89917 89931 89935 89937 89939 89941 89943 89949 89955 89965 89971 89977 89997 90037 90095 90141 90151 90161 90163 90165 90169 90173 90181 90447 90471 90499 90631 90637 90655 90757 90789 90795 90797 90811 90823 90843 90845 90849 90855 90865 90879 90903 90999 91055 91069 91213 91765 91767 91779 91785 91791 91797 91803 91809 91811 91815 91825 91827 91829 91833 91841 91855 91863 91865 91867 91871 91873 91879 91883 91895 91897 91899 91905 91931 91947 91949 91957 91961 91963 91967 91969 91975 91977 91979 91981 91983 91985 91987 91991 91995 92007 92009 92017 92021 92025 92027 92031 92047 92055 92063 92067 92085 92091 92097 92101 92103 92105 92107 92113 92115 92123 92127 92133 92135 92163 92165 92167 92169 92177 92193 92195 92197 92203 92209 92211 92215 92217 92219 92231 92249 92251 92253 92257 92265 92271 92293 92305 92317 92319 92321 92333 92349 92357 92371 92387 92403 92413 92417 92429 92445 92453 92469 92471 92479 92491 92495 92499 92505 92515 92535 92541 92547 92567 92593 92595 92599 92615 92623 92627 92635 92637 92651 92653 92661 92677 92679 92699 92701 92707 92715 92727 92733 92737 92751 92757 92771 92793 92829 92845 92847 92857 92859 92861 92863 92865 92867 92869 92871 92875 92877 92879 92887 92889 92905 92927 92929 92931 92933 92953 92969 92973 92975 92977 92983 93015 93023 93025 93027 93031 93033 93035 93037 93039 93041 93043 93045 93047 93049 93053 93055 93057 93061 93063 93065 93067 93069 93071 93073) for request_id in "${request_ids[@]}"; do echo idds resume_requests --request_id=${request_id} idds resume_requests --request_id=${request_id} diff --git a/main/lib/idds/tests/run_sql.py b/main/lib/idds/tests/run_sql.py index ac327154..0b85a328 100644 --- a/main/lib/idds/tests/run_sql.py +++ b/main/lib/idds/tests/run_sql.py @@ -23,7 +23,8 @@ def get_subfinished_requests(db_pool): connection = db_pool.acquire() req_ids = [] - sql = """select request_id from atlas_IDDS.requests where status in (4,5) and scope!='hpo'""" + # sql = """select request_id from atlas_IDDS.requests where status in (4,5) and scope!='hpo'""" + sql = """select request_id from atlas_IDDS.requests where scope!='hpo' and ( status in (4,5) or request_id in (select request_id from atlas_idds.transforms where status in (4, 5) and transform_type=2)) order by request_id""" cursor = connection.cursor() cursor.execute(sql) rows = cursor.fetchall() @@ -34,6 +35,7 @@ def get_subfinished_requests(db_pool): connection.commit() db_pool.release(connection) + print(len(req_ids)) print(req_ids) diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 6fe421e5..8534115e 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index b9ee9fb8..de0c004f 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.1 - - idds-workflow==0.7.1 - - idds-client==0.7.1 \ No newline at end of file + - idds-common==0.7.2 + - idds-workflow==0.7.2 + - idds-client==0.7.2 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 6fe421e5..8534115e 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/website/version.py b/website/version.py index 6fe421e5..8534115e 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 6fe421e5..8534115e 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.1" +release_version = "0.7.2" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index eafe21aa..857553f8 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.1 \ No newline at end of file + - idds-common==0.7.2 \ No newline at end of file From 0d8f0900e40d28d21cb833e21bc3dd53039e19ef Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 25 Aug 2021 17:31:38 +0200 Subject: [PATCH 051/156] new archive requests table --- main/etc/sql/oracle_11.sql | 35 +++++++++++++++++++++++++++++++++++ main/etc/sql/oracle_19.sql | 35 +++++++++++++++++++++++++++++++++++ monitor/conf.js | 12 ++++++------ 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/main/etc/sql/oracle_11.sql b/main/etc/sql/oracle_11.sql index 93ee285a..319bde25 100644 --- a/main/etc/sql/oracle_11.sql +++ b/main/etc/sql/oracle_11.sql @@ -447,6 +447,41 @@ CREATE OR REPLACE TRIGGER TRIG_HEALTH_ID / +--- request archive table +CREATE TABLE REQUESTS_archive +( + request_id NUMBER(12), + scope VARCHAR2(25), + name VARCHAR2(255), + requester VARCHAR2(20), + request_type NUMBER(2), + transform_tag VARCHAR2(10), + workload_id NUMBER(10), + priority NUMBER(7), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + created_at DATE, + updated_at DATE, + next_poll_at DATE, + accessed_at DATE, + expired_at DATE, + errors VARCHAR2(1024), + request_metadata CLOB, + processing_metadata, + CONSTRAINT REQ_AR_PK PRIMARY KEY (request_id) USING INDEX LOCAL + --- CONSTRAINT REQUESTS_NAME_SCOPE_UQ UNIQUE (name, scope, requester, request_type, transform_tag, workload_id) -- USING INDEX LOCAL, +) +PCTFREE 3 +PARTITION BY RANGE(REQUEST_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + +CREATE INDEX REQ_AR_SCOPE_NAME_IDX ON REQUESTS_archive (name, scope, workload_id) LOCAL; +--- drop index REQUESTS_STATUS_PRIORITY_IDX +CREATE INDEX REQ_AR_STATUS_PRIORITY_IDX ON REQUESTS_archive (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; + + SELECT cols.table_name, cols.column_name, cols.position, cons.status, cons.owner FROM all_constraints cons, all_cons_columns cols WHERE cols.table_name = 'HEALTH' diff --git a/main/etc/sql/oracle_19.sql b/main/etc/sql/oracle_19.sql index 346f103d..c3650c04 100644 --- a/main/etc/sql/oracle_19.sql +++ b/main/etc/sql/oracle_19.sql @@ -341,3 +341,38 @@ CREATE TABLE HEALTH CONSTRAINT HEALTH_PK PRIMARY KEY (health_id), -- USING INDEX LOCAL, CONSTRAINT HEALTH_UQ UNIQUE (agent, hostname, pid, thread_id) ); + + +--- request archive table +CREATE TABLE REQUESTS_archive +( + request_id NUMBER(12), + scope VARCHAR2(25), + name VARCHAR2(255), + requester VARCHAR2(20), + request_type NUMBER(2), + transform_tag VARCHAR2(10), + workload_id NUMBER(10), + priority NUMBER(7), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + created_at DATE, + updated_at DATE, + next_poll_at DATE, + accessed_at DATE, + expired_at DATE, + errors VARCHAR2(1024), + request_metadata CLOB, + processing_metadata, + CONSTRAINT REQ_AR_PK PRIMARY KEY (request_id) USING INDEX LOCAL + --- CONSTRAINT REQUESTS_NAME_SCOPE_UQ UNIQUE (name, scope, requester, request_type, transform_tag, workload_id) -- USING INDEX LOCAL, +) +PCTFREE 3 +PARTITION BY RANGE(REQUEST_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + +CREATE INDEX REQ_AR_SCOPE_NAME_IDX ON REQUESTS_archive (name, scope, workload_id) LOCAL; +--- drop index REQUESTS_STATUS_PRIORITY_IDX +CREATE INDEX REQ_AR_STATUS_PRIORITY_IDX ON REQUESTS_archive (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; diff --git a/monitor/conf.js b/monitor/conf.js index b0e6853c..66db231f 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus7110.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus7110.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus7110.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus7110.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus7110.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus7110.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus728.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus728.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus728.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/false/true" } From d9758a53e9d52d2ff2c1ffef64aa2ae4238e76b6 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 14:23:29 +0200 Subject: [PATCH 052/156] add archive function --- main/etc/sql/oracle_11.sql | 160 +++++++++++++++++++ main/etc/sql/oracle_19.sql | 165 +++++++++++++++++++- main/lib/idds/agents/archive/run_archive.py | 146 +++++++++++++++++ 3 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 main/lib/idds/agents/archive/run_archive.py diff --git a/main/etc/sql/oracle_11.sql b/main/etc/sql/oracle_11.sql index 319bde25..83c1188a 100644 --- a/main/etc/sql/oracle_11.sql +++ b/main/etc/sql/oracle_11.sql @@ -482,6 +482,166 @@ CREATE INDEX REQ_AR_SCOPE_NAME_IDX ON REQUESTS_archive (name, scope, workload_id CREATE INDEX REQ_AR_STATUS_PRIORITY_IDX ON REQUESTS_archive (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; +CREATE TABLE TRANSFORMS_archive +( + transform_id NUMBER(12), + request_id NUMBER(12), + workload_id NUMBER(10), + transform_type NUMBER(2), + transform_tag VARCHAR2(20), + priority NUMBER(7), + safe2get_output_from_input NUMBER(10), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + retries NUMBER(5), + created_at DATE, + updated_at DATE, + next_poll_at DATE, + started_at DATE, + finished_at DATE, + expired_at DATE, + transform_metadata CLOB, + running_metadata CLOB, + CONSTRAINT TF_AR_PK PRIMARY KEY (transform_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE PROCESSINGS_archive +( + processing_id NUMBER(12), + transform_id NUMBER(12) Not NULL, + request_id NUMBER(12), + workload_id NUMBER(10), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + submitter VARCHAR2(20), + submitted_id NUMBER(12), + granularity NUMBER(10), + granularity_type NUMBER(2), + created_at DATE, + updated_at DATE, + next_poll_at DATE, + submitted_at DATE, + finished_at DATE, + expired_at DATE, + processing_metadata CLOB, + running_metadata CLOB, + output_metadata CLOB, + CONSTRAINT PR_AR_PK PRIMARY KEY (processing_id), + CONSTRAINT PR_AR_TRANSFORM_ID_FK FOREIGN KEY(transform_id) REFERENCES TRANSFORMS_archive(transform_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE COLLECTIONS_archive +( + coll_id NUMBER(14), + coll_type NUMBER(2), + transform_id NUMBER(12), + request_id NUMBER(12), + workload_id NUMBER(10), + relation_type NUMBER(2), + scope VARCHAR2(25), + name VARCHAR2(255), + bytes NUMBER(19), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + total_files NUMBER(19), + storage_id NUMBER(10), + new_files NUMBER(10), + processed_files NUMBER(10), + processing_files NUMBER(10), + processing_id NUMBER(12), + retries NUMBER(5) DEFAULT 0, + created_at DATE, + updated_at DATE, + next_poll_at DATE, + accessed_at DATE, + expired_at DATE, + coll_metadata CLOB, + CONSTRAINT CL_AR_PK PRIMARY KEY (coll_id), -- USING INDEX LOCAL, + CONSTRAINT CL_AR_NAME_SCOPE_UQ UNIQUE (name, scope, transform_id, relation_type), -- USING INDEX LOCAL, + CONSTRAINT CL_AR_TRANSFORM_ID_FK FOREIGN KEY(transform_id) REFERENCES TRANSFORMS_archive(transform_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE CONTENTS_archive +( + content_id NUMBER(12), + transform_id NUMBER(12), + coll_id NUMBER(14), + request_id NUMBER(12), + workload_id NUMBER(10), + map_id NUMBER(12), + scope VARCHAR2(25), + name VARCHAR2(255), + min_id NUMBER(7), + max_id NUMBER(7), + content_type NUMBER(2), + content_relation_type NUMBER(2), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + bytes NUMBER(12), + md5 VARCHAR2(32), + adler32 VARCHAR2(8), + processing_id NUMBER(12), + storage_id NUMBER(10), + retries NUMBER(5) DEFAULT 0, + path VARCHAR2(4000), + created_at DATE, + updated_at DATE, + accessed_at DATE, + expired_at DATE, + content_metadata VARCHAR2(100), + CONSTRAINT CT_AR_PK PRIMARY KEY (content_id), + --- CONSTRAINT CT_ID_UQ UNIQUE (transform_id, coll_id, map_id, name, min_id, max_id) USING INDEX LOCAL, + CONSTRAINT CT_AR_TRANSFORM_ID_FK FOREIGN KEY(transform_id) REFERENCES TRANSFORMS_archive(transform_id), + CONSTRAINT CT_AR_COLL_ID_FK FOREIGN KEY(coll_id) REFERENCES COLLECTIONS_archive(coll_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE MESSAGES_archive +( + msg_id NUMBER(12), + msg_type NUMBER(2), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + source NUMBER(2), + destination NUMBER(2), + request_id NUMBER(12), + workload_id NUMBER(10), + transform_id NUMBER(12), + processing_id NUMBER(12), + num_contents NUMBER(7), + created_at DATE, + updated_at DATE, + msg_content CLOB, + CONSTRAINT MG_AR_PK PRIMARY KEY (msg_id) -- USING INDEX LOCAL, +) +PCTFREE 3 +PARTITION BY RANGE(REQUEST_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); SELECT cols.table_name, cols.column_name, cols.position, cons.status, cons.owner FROM all_constraints cons, all_cons_columns cols WHERE cols.table_name = 'HEALTH' diff --git a/main/etc/sql/oracle_19.sql b/main/etc/sql/oracle_19.sql index c3650c04..ba3dafeb 100644 --- a/main/etc/sql/oracle_19.sql +++ b/main/etc/sql/oracle_19.sql @@ -364,7 +364,7 @@ CREATE TABLE REQUESTS_archive expired_at DATE, errors VARCHAR2(1024), request_metadata CLOB, - processing_metadata, + processing_metadata CLOB, CONSTRAINT REQ_AR_PK PRIMARY KEY (request_id) USING INDEX LOCAL --- CONSTRAINT REQUESTS_NAME_SCOPE_UQ UNIQUE (name, scope, requester, request_type, transform_tag, workload_id) -- USING INDEX LOCAL, ) @@ -376,3 +376,166 @@ INTERVAL ( 100000 ) CREATE INDEX REQ_AR_SCOPE_NAME_IDX ON REQUESTS_archive (name, scope, workload_id) LOCAL; --- drop index REQUESTS_STATUS_PRIORITY_IDX CREATE INDEX REQ_AR_STATUS_PRIORITY_IDX ON REQUESTS_archive (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; + + +CREATE TABLE TRANSFORMS_archive +( + transform_id NUMBER(12), + request_id NUMBER(12), + workload_id NUMBER(10), + transform_type NUMBER(2), + transform_tag VARCHAR2(20), + priority NUMBER(7), + safe2get_output_from_input NUMBER(10), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + retries NUMBER(5), + created_at DATE, + updated_at DATE, + next_poll_at DATE, + started_at DATE, + finished_at DATE, + expired_at DATE, + transform_metadata CLOB, + running_metadata CLOB, + CONSTRAINT TF_AR_PK PRIMARY KEY (transform_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE PROCESSINGS_archive +( + processing_id NUMBER(12), + transform_id NUMBER(12) Not NULL, + request_id NUMBER(12), + workload_id NUMBER(10), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + submitter VARCHAR2(20), + submitted_id NUMBER(12), + granularity NUMBER(10), + granularity_type NUMBER(2), + created_at DATE, + updated_at DATE, + next_poll_at DATE, + submitted_at DATE, + finished_at DATE, + expired_at DATE, + processing_metadata CLOB, + running_metadata CLOB, + output_metadata CLOB, + CONSTRAINT PR_AR_PK PRIMARY KEY (processing_id), + CONSTRAINT PR_AR_TRANSFORM_ID_FK FOREIGN KEY(transform_id) REFERENCES TRANSFORMS_archive(transform_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE COLLECTIONS_archive +( + coll_id NUMBER(14), + coll_type NUMBER(2), + transform_id NUMBER(12), + request_id NUMBER(12), + workload_id NUMBER(10), + relation_type NUMBER(2), + scope VARCHAR2(25), + name VARCHAR2(255), + bytes NUMBER(19), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + total_files NUMBER(19), + storage_id NUMBER(10), + new_files NUMBER(10), + processed_files NUMBER(10), + processing_files NUMBER(10), + processing_id NUMBER(12), + retries NUMBER(5) DEFAULT 0, + created_at DATE, + updated_at DATE, + next_poll_at DATE, + accessed_at DATE, + expired_at DATE, + coll_metadata CLOB, + CONSTRAINT CL_AR_PK PRIMARY KEY (coll_id), -- USING INDEX LOCAL, + CONSTRAINT CL_AR_NAME_SCOPE_UQ UNIQUE (name, scope, transform_id, relation_type), -- USING INDEX LOCAL, + CONSTRAINT CL_AR_TRANSFORM_ID_FK FOREIGN KEY(transform_id) REFERENCES TRANSFORMS_archive(transform_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + + +CREATE TABLE CONTENTS_archive +( + content_id NUMBER(12), + transform_id NUMBER(12), + coll_id NUMBER(14), + request_id NUMBER(12), + workload_id NUMBER(10), + map_id NUMBER(12), + scope VARCHAR2(25), + name VARCHAR2(255), + min_id NUMBER(7), + max_id NUMBER(7), + content_type NUMBER(2), + content_relation_type NUMBER(2), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + bytes NUMBER(12), + md5 VARCHAR2(32), + adler32 VARCHAR2(8), + processing_id NUMBER(12), + storage_id NUMBER(10), + retries NUMBER(5) DEFAULT 0, + path VARCHAR2(4000), + created_at DATE, + updated_at DATE, + accessed_at DATE, + expired_at DATE, + content_metadata VARCHAR2(100), + CONSTRAINT CT_AR_PK PRIMARY KEY (content_id), + --- CONSTRAINT CT_ID_UQ UNIQUE (transform_id, coll_id, map_id, name, min_id, max_id) USING INDEX LOCAL, + CONSTRAINT CT_AR_TRANSFORM_ID_FK FOREIGN KEY(transform_id) REFERENCES TRANSFORMS_archive(transform_id), + CONSTRAINT CT_AR_COLL_ID_FK FOREIGN KEY(coll_id) REFERENCES COLLECTIONS_archive(coll_id) +) +PCTFREE 3 +PARTITION BY RANGE(TRANSFORM_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); + + +CREATE TABLE MESSAGES_archive +( + msg_id NUMBER(12), + msg_type NUMBER(2), + status NUMBER(2), + substatus NUMBER(2), + locking NUMBER(2), + source NUMBER(2), + destination NUMBER(2), + request_id NUMBER(12), + workload_id NUMBER(10), + transform_id NUMBER(12), + processing_id NUMBER(12), + num_contents NUMBER(7), + created_at DATE, + updated_at DATE, + msg_content CLOB, + CONSTRAINT MG_AR_PK PRIMARY KEY (msg_id) -- USING INDEX LOCAL, +) +PCTFREE 3 +PARTITION BY RANGE(REQUEST_ID) +INTERVAL ( 100000 ) +( PARTITION initial_part VALUES LESS THAN (1) ); diff --git a/main/lib/idds/agents/archive/run_archive.py b/main/lib/idds/agents/archive/run_archive.py new file mode 100644 index 00000000..97614d20 --- /dev/null +++ b/main/lib/idds/agents/archive/run_archive.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 + + +""" +performance test to insert contents. +""" +import cx_Oracle + + +from idds.common.config import config_get +# from idds.core.contents import add_content + + +def get_archive_sql(schema): + sql = """ + BEGIN + FOR i in (SELECT request_id, scope, name, requester, request_type, transform_tag, + workload_id, priority, status, substatus, locking, created_at, + updated_at, next_poll_at, accessed_at, expired_at, errors, + request_metadata, processing_metadata + FROM {schema}.requests + WHERE status in (3, 5, 17) and created_at < sysdate - interval '9' month + order by request_id asc) + LOOP + --- archive records + insert into {schema}.requests_archive(request_id, scope, name, requester, + request_type, transform_tag, workload_id, priority, status, + substatus, locking, created_at, updated_at, next_poll_at, + accessed_at, expired_at, errors, request_metadata, + processing_metadata) + values(i.request_id, i.scope, i.name, i.requester, i.request_type, i.transform_tag, + i.workload_id, i.priority, i.status, i.substatus, i.locking, i.created_at, + i.updated_at, i.next_poll_at, i.accessed_at, i.expired_at, i.errors, + i.request_metadata, i.processing_metadata); + + insert into {schema}.transforms_archive(transform_id, request_id, workload_id, + transform_type, transform_tag, priority, + safe2get_output_from_input, status, + substatus, locking, retries, created_at, + updated_at, next_poll_at, started_at, + finished_at, expired_at, transform_metadata, + running_metadata) + select transform_id, request_id, workload_id, transform_type, transform_tag, + priority, safe2get_output_from_input, status, substatus, locking, + retries, created_at, updated_at, next_poll_at, started_at, finished_at, + expired_at, transform_metadata, running_metadata + from {schema}.transforms where request_id=i.request_id; + + insert into {schema}.processings_archive(processing_id, transform_id, request_id, + workload_id, status, substatus, locking, submitter, submitted_id, + granularity, granularity_type, created_at, updated_at, next_poll_at, + submitted_at, finished_at, expired_at, processing_metadata, + running_metadata, output_metadata) + select processing_id, transform_id, request_id, workload_id, status, substatus, + locking, submitter, submitted_id, granularity, granularity_type, + created_at, updated_at, next_poll_at, submitted_at, finished_at, expired_at, + processing_metadata, running_metadata, output_metadata + from {schema}.processings where request_id=i.request_id; + + insert into {schema}.collections_archive(coll_id, coll_type, transform_id, request_id, + workload_id, relation_type, scope, name, bytes, status, substatus, locking, + total_files, storage_id, new_files, processed_files, processing_files, + processing_id, retries, created_at, updated_at, next_poll_at, accessed_at, + expired_at, coll_metadata) + select coll_id, coll_type, transform_id, request_id, workload_id, relation_type, + scope, name, bytes, status, substatus, locking, total_files, storage_id, + new_files, processed_files, processing_files, processing_id, retries, + created_at, updated_at, next_poll_at, accessed_at, expired_at, + coll_metadata + from {schema}.collections where request_id=i.request_id; + + insert into {schema}.contents_archive(content_id, transform_id, coll_id, request_id, + workload_id, map_id, scope, name, min_id, max_id, content_type, + content_relation_type, status, substatus, locking, bytes, md5, adler32, + processing_id, storage_id, retries, path, created_at, updated_at, + accessed_at, expired_at, content_metadata) + select content_id, transform_id, coll_id, request_id, workload_id, map_id, + scope, name, min_id, max_id, content_type, content_relation_type, + status, substatus, locking, bytes, md5, adler32, processing_id, + storage_id, retries, path, created_at, updated_at, accessed_at, + expired_at, content_metadata + from {schema}.contents where request_id=i.request_id; + + insert into {schema}.messages_archive(msg_id, msg_type, status, substatus, locking, + source, destination, request_id, workload_id, transform_id, processing_id, + num_contents, created_at, updated_at, msg_content) + select msg_id, msg_type, status, substatus, locking, source, destination, + request_id, workload_id, transform_id, processing_id, num_contents, + created_at, updated_at, msg_content + from {schema}.messages where request_id=i.request_id; + + + -- clean records + delete from {schema}.messages where request_id = i.request_id; + delete from {schema}.contents where request_id = i.request_id; + delete from {schema}.collections where request_id = i.request_id; + delete from {schema}.processings where request_id = i.request_id; + delete from {schema}.transforms where request_id = i.request_id; + delete from {schema}.requests where request_id = i.request_id; + END LOOP; + COMMIT; + END; + """ + sql = sql.format(schema=schema) + return sql + + +def run_archive_sql(db_pool, schema): + connection = db_pool.acquire() + + sql = get_archive_sql(schema) + # print(sql) + cursor = connection.cursor() + cursor.execute(sql) + cursor.close() + + connection.commit() + db_pool.release(connection) + + +def get_session_pool(): + sql_connection = config_get('database', 'default') + sql_connection = sql_connection.replace("oracle://", "") + user_pass, tns = sql_connection.split('@') + user, passwd = user_pass.split(':') + db_pool = cx_Oracle.SessionPool(user, passwd, tns, min=12, max=20, increment=1) + + schema = config_get('database', 'schema') + return db_pool, schema + + +def run_archive(): + pool, schema = get_session_pool() + run_archive_sql(pool, schema) + + +if __name__ == '__main__': + run_archive() From f1e5fce6c186597c4fb36235986f27bb867d0ecc Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 14:24:16 +0200 Subject: [PATCH 053/156] fix failed transforms --- .../idds/agents/transformer/transformer.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index a622b0bc..14cd52bf 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -381,7 +381,13 @@ def process_new_transform(self, transform): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - transform_parameters = {'status': TransformStatus.Failed, + if transform['retries'] > 10: + tf_status = TransformStatus.Failed + else: + tf_status = TransformStatus.Transforming + transform_parameters = {'status': tf_status, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4), + 'retries': transform['retries'] + 1, 'locking': TransformLocking.Idle} ret = {'transform': transform, 'transform_parameters': transform_parameters} return ret @@ -1019,9 +1025,15 @@ def process_running_transform_message(self, transform, messages): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + if transform['retries'] > 10: + tf_status = TransformStatus.Failed + else: + tf_status = TransformStatus.Transforming ret = {'transform': transform, - 'transform_parameters': {'status': TransformStatus.Failed, + 'transform_parameters': {'status': tf_status, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4), 'locking': TransformLocking.Idle, + 'retries': transform['retries'] + 1, 'errors': {'msg': '%s: %s' % (ex, traceback.format_exc())}}} return ret @@ -1040,7 +1052,13 @@ def process_running_transform(self, transform): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - transform_parameters = {'status': TransformStatus.Failed, + if transform['retries'] > 10: + tf_status = TransformStatus.Failed + else: + tf_status = TransformStatus.Transforming + transform_parameters = {'status': tf_status, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4), + 'retries': transform['retries'] + 1, 'locking': TransformLocking.Idle} ret = {'transform': transform, 'transform_parameters': transform_parameters} return ret From dbd674f585c3b6dd434c7bf0124fe8d83c64f690 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 14:27:33 +0200 Subject: [PATCH 054/156] fix failed transforms --- main/lib/idds/agents/transformer/transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 14cd52bf..d17fd0d2 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -878,6 +878,7 @@ def process_running_transform_real(self, transform): work.tosuspend = True elif transform['status'] in [TransformStatus.ToResume]: transform['status'] = TransformStatus.Resuming + transform['retries'] = 0 work.toresume = True to_resume_transform = True reactivated_contents = self.reactive_contents(registered_input_output_maps) @@ -977,6 +978,7 @@ def process_running_transform_real(self, transform): 'locking': TransformLocking.Idle, 'workload_id': transform['workload_id'], 'next_poll_at': next_poll_at, + 'retries': transform['retries'], 'transform_metadata': transform['transform_metadata']} # if transform_substatus: # transform_parameters['substatus'] = transform_substatus From dc7cd1e4b892673cc7d55504ecd542ffab6d2034 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 14:32:08 +0200 Subject: [PATCH 055/156] new version 0.7.3 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 8534115e..8e347f2a 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 62c0b081..942e0228 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.2 - - idds-workflow==0.7.2 \ No newline at end of file + - idds-common==0.7.3 + - idds-workflow==0.7.3 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 8534115e..8e347f2a 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 4be5b6ce..11382603 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.2 - - idds-workflow==0.7.2 \ No newline at end of file + - idds-common==0.7.3 + - idds-workflow==0.7.3 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 8534115e..8e347f2a 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 2470e47f..8d90eabd 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 659912ec..f32d046c 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.2 - - idds-workflow==0.7.2 \ No newline at end of file + - idds-common==0.7.3 + - idds-workflow==0.7.3 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 8534115e..8e347f2a 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index de0c004f..05a9997c 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.2 - - idds-workflow==0.7.2 - - idds-client==0.7.2 \ No newline at end of file + - idds-common==0.7.3 + - idds-workflow==0.7.3 + - idds-client==0.7.3 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 8534115e..8e347f2a 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/website/version.py b/website/version.py index 8534115e..8e347f2a 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 8534115e..8e347f2a 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.2" +release_version = "0.7.3" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 857553f8..f344dcf8 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.2 \ No newline at end of file + - idds-common==0.7.3 \ No newline at end of file From d2e9756c0ab7ca2625d8bd0dc0bc13a73b92f87f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 15:36:21 +0200 Subject: [PATCH 056/156] fix to catch not found rule --- atlas/lib/idds/atlas/workflow/atlasstageinwork.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py index 03c65960..64b2fd27 100644 --- a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py @@ -375,6 +375,11 @@ def poll_processing_updates(self, processing, input_output_maps): proc.has_new_updates() return update_processing, updated_contents, {} + except exceptions.ProcessNotFound as ex: + self.logger.warn("processing_id %s not not found: %s" % (processing['processing_id'], str(ex))) + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.SubFinished}} + return update_processing, [], {} except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) From 8511fb904c26d214a9a842ed52d97e2b610b73c5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 15:49:01 +0200 Subject: [PATCH 057/156] count processing retires --- atlas/lib/idds/atlas/workflow/atlashpowork.py | 93 ++++++++++--------- main/lib/idds/agents/archive/run_archive.py | 2 +- .../tests/resume_subfinished_data_carousel.sh | 3 +- monitor/conf.js | 12 +-- workflow/lib/idds/workflow/work.py | 10 ++ 5 files changed, 69 insertions(+), 51 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlashpowork.py b/atlas/lib/idds/atlas/workflow/atlashpowork.py index 0a34c2c8..be85c682 100644 --- a/atlas/lib/idds/atlas/workflow/atlashpowork.py +++ b/atlas/lib/idds/atlas/workflow/atlashpowork.py @@ -658,51 +658,60 @@ def parse_processing_outputs(self, processing): return None, 'Failed to load the content of %s: %s' % (str(full_output_json), str(ex)) def poll_processing(self, processing): - proc = processing['processing_metadata']['processing'] - job_status, job_err_msg = self.poll_condor_job_status(processing, proc.external_id) - processing_outputs = None - reset_expired_at = False - if job_status in [ProcessingStatus.Finished]: - job_outputs, parser_errors = self.parse_processing_outputs(processing) - if job_outputs: - processing_status = ProcessingStatus.Finished + try: + proc = processing['processing_metadata']['processing'] + job_status, job_err_msg = self.poll_condor_job_status(processing, proc.external_id) + processing_outputs = None + reset_expired_at = False + if job_status in [ProcessingStatus.Finished]: + job_outputs, parser_errors = self.parse_processing_outputs(processing) + if job_outputs: + processing_status = ProcessingStatus.Finished + processing_err = None + processing_outputs = job_outputs + else: + processing_status = ProcessingStatus.Failed + processing_err = parser_errors + elif job_status in [ProcessingStatus.Failed]: + processing_status = job_status + processing_err = job_err_msg + elif self.toexpire: + processing_status = ProcessingStatus.Expired + processing_err = "The processing is expired" + elif job_status in [ProcessingStatus.Cancelled]: + processing_status = job_status + processing_err = job_err_msg + elif self.tocancel: + self.cancelled_processings.append(proc.internal_id) + processing_status = ProcessingStatus.Cancelled + processing_outputs = None + processing_err = 'Cancelled' + elif self.tosuspend: + self.suspended_processings.append(proc.internal_id) + processing_status = ProcessingStatus.Suspended + processing_outputs = None + processing_err = 'Suspend' + elif self.toresume: + # self.old_processings.append(processing['processing_metadata']['internal_id']) + # self.active_processings.clear() + # self.active_processings.remove(processing['processing_metadata']['internal_id']) + processing['processing_metadata']['resuming_at'] = datetime.datetime.utcnow() + processing_status = ProcessingStatus.Running + reset_expired_at = True + processing_outputs = None processing_err = None - processing_outputs = job_outputs else: + processing_status = job_status + processing_err = job_err_msg + return processing_status, processing_outputs, processing_err, reset_expired_at + except Exception as ex: + self.logger.error("processing_id %s exception: %s, %s" % (processing['processing_id'], str(ex), traceback.format_exc())) + proc.retries += 1 + if proc.retries > 10: processing_status = ProcessingStatus.Failed - processing_err = parser_errors - elif job_status in [ProcessingStatus.Failed]: - processing_status = job_status - processing_err = job_err_msg - elif self.toexpire: - processing_status = ProcessingStatus.Expired - processing_err = "The processing is expired" - elif job_status in [ProcessingStatus.Cancelled]: - processing_status = job_status - processing_err = job_err_msg - elif self.tocancel: - self.cancelled_processings.append(proc.internal_id) - processing_status = ProcessingStatus.Cancelled - processing_outputs = None - processing_err = 'Cancelled' - elif self.tosuspend: - self.suspended_processings.append(proc.internal_id) - processing_status = ProcessingStatus.Suspended - processing_outputs = None - processing_err = 'Suspend' - elif self.toresume: - # self.old_processings.append(processing['processing_metadata']['internal_id']) - # self.active_processings.clear() - # self.active_processings.remove(processing['processing_metadata']['internal_id']) - processing['processing_metadata']['resuming_at'] = datetime.datetime.utcnow() - processing_status = ProcessingStatus.Running - reset_expired_at = True - processing_outputs = None - processing_err = None - else: - processing_status = job_status - processing_err = job_err_msg - return processing_status, processing_outputs, processing_err, reset_expired_at + else: + processing_status = ProcessingStatus.Running + return processing_status, None, None, False def poll_processing_updates(self, processing, input_output_maps): processing_status, processing_outputs, processing_err, reset_expired_at = self.poll_processing(processing) diff --git a/main/lib/idds/agents/archive/run_archive.py b/main/lib/idds/agents/archive/run_archive.py index 97614d20..486c4e6f 100644 --- a/main/lib/idds/agents/archive/run_archive.py +++ b/main/lib/idds/agents/archive/run_archive.py @@ -27,7 +27,7 @@ def get_archive_sql(schema): updated_at, next_poll_at, accessed_at, expired_at, errors, request_metadata, processing_metadata FROM {schema}.requests - WHERE status in (3, 5, 17) and created_at < sysdate - interval '9' month + WHERE status in (3, 5, 9, 17) and created_at < sysdate - interval '7' month order by request_id asc) LOOP --- archive records diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh index e921db6b..39e55bc6 100644 --- a/main/lib/idds/tests/resume_subfinished_data_carousel.sh +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -4,9 +4,8 @@ request_ids=(75039 75065 75569 75683 76087 76181 76211 76455 76727 77281 77283 7 request_ids=(75039 77281 77283 77285 77287 83271 83483 77279 77367 77847 77947 78123 78205 80547 80827 82153) -request_ids=(75039 77281 77283 77285 77287 78533 79075 79587 80923 82301 82731 83069 83119 83271 83483 83879 83887 83891 83893 83897 83899 83905 83907 83909 83915 83921 83925 83927 83933 83939 83943 83945 83949 83955 83959 83961 83963 83967 83977 83979 83981 83983 83985 83993 83995 84067 84081 84083 84091 84095 84109 84115 84121 84125 84133 84135 84145 84147 84153 84161 84163 84167 84169 84179 84181 84185 84187 84191 84195 84199 84275 84319 84325 84399 84569 85597 86831 87541 88187 88695 89583 73 102 103 104 105 106 706 707 708 710 711 712 713 714 715 716 718 719 720 722 723 725 726 727 728 1783 1785 1787 2039 2041 2045 2047 2059 2077 2093 2119 2127 2131 2149 2155 2167 2225 2231 2233 2237 2263 2265 2267 2279 7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 16539 16541 16543 16545 16547 16549 16825 17615 19289 19319 19321 19397 19569 19583 19711 20017 20019 20115 20117 20199 20201 20203 20207 20243 21207 21487 21497 21499 21647 21673 21675 21677 21689 21699 21709 21711 21715 21717 21719 21721 21723 21725 21727 21731 21743 21761 21773 21777 21919 21923 21925 21929 21931 22037 22185 22187 22189 22193 22195 22197 22199 22201 22223 22225 22227 22285 22287 22305 22307 22309 22351 22353 22355 22357 22359 22449 22451 22453 22455 22457 22459 22461 22463 22465 22467 22469 22471 24757 24759 24761 24763 24765 24767 24769 24771 24873 24875 24877 24879 24881 24883 24885 25423 25425 25427 25429 25431 25433 25435 25437 25439 25441 25443 25445 25447 25449 25451 25453 25455 25457 25459 25461 27171 27173 27183 27185 27187 27189 27191 27193 27195 27197 27199 27201 27203 27205 27207 27209 27211 27213 27215 27217 27219 27221 27641 27643 27645 27647 27649 27651 27653 27655 27657 27659 27661 27663 27665 27667 27669 27671 27673 27675 27677 27679 27975 27977 27979 27981 27983 27985 27987 27989 27991 27993 27995 27997 27999 28001 28003 28005 28007 28009 28011 28013 29343 29345 29347 29349 29351 29353 29355 29357 29359 29361 29363 29365 29367 29369 29371 29373 29375 29377 29379 29381 29385 29639 29641 29643 29645 29647 29649 29651 29653 29655 29657 29659 29661 29663 29665 29667 29669 29671 29673 29675 29677 31559 31561 31963 31965 31967 32091 32117 32119 32121 32123 32163 32187 32189 32193 32195 32197 32207 32319 32327 32335 32337 32341 32347 32367 32541 32585 32587 36715 36717 36727 36835 36837 36839 36841 36843 36845 36847 36849 36851 36853 36855 36857 36859 36861 36863 36865 36867 36869 36871 36873 37155 37157 37159 37161 37163 37165 37167 37169 37171 37173 37175 37177 37179 37181 37183 37185 37187 37191 37193 37195 43907 43909 43911 77279 77367 77847 77947 78123 78205 80547 80827 82153 87115 87137 87199 87661 87715) +request_ids=(7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 16539 16541 16543 16545 16547 16549 16825 17615 19289 19319 19321 19397 75039 77279 77281 77283 77285 77287 77947 80109 82731 83069 83925 83933 83943 83945 84081 84083 84091 84125 84145 84181 84199 84569 87199 87661 87715 88841 89579 89891 89917 89937 89939 89941 89949 89955 89997 90165 90471 90795 90855 91069 91765 91779 91797 91803 91841 91865 91867 91879 91949 91961 91969 91975 91981 91983 91985 92009 92017 92021 92025 92027 92031 92055 92063 92097 92105 92107 92113 92133 92163 92165 92169 92197 92203 92209 92211 92231 92265 92305 92319 92387 92417 92445 92453 92495 92499 92535 92595 92623 92757 92861 92865 92871 92879 92889 92975 92983 93023 93031 93033 93049) -request_ids=(73 102 103 104 105 106 706 707 708 710 711 712 713 714 715 716 718 719 720 722 723 725 726 727 728 1783 1785 1787 2039 2041 2045 2047 2059 2077 2093 2119 2127 2131 2149 2155 2167 2225 2231 2233 2237 2263 2265 2267 2279 7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 16539 16541 16543 16545 16547 16549 16825 17615 19289 19319 19321 19397 75039 77279 77281 77283 77285 77287 77367 77847 77947 78123 78205 78533 79075 79587 80109 80547 80827 80923 82153 82301 82731 83069 83119 83271 83483 83879 83887 83891 83893 83897 83915 83921 83925 83927 83933 83939 83943 83945 83949 83955 83959 83961 83963 83967 83977 83979 83981 83985 83993 83995 84067 84081 84083 84091 84095 84109 84115 84121 84125 84133 84135 84145 84147 84153 84161 84163 84167 84169 84179 84181 84185 84187 84191 84195 84199 84275 84319 84325 84335 84399 84569 85597 86831 87115 87137 87199 87541 87661 87715 87881 87993 88037 88187 88283 88353 88361 88695 88839 88841 88847 88849 88893 88991 89073 89415 89579 89583 89885 89887 89891 89917 89931 89935 89937 89939 89941 89943 89949 89955 89965 89971 89977 89997 90037 90095 90141 90151 90161 90163 90165 90169 90173 90181 90447 90471 90499 90631 90637 90655 90757 90789 90795 90797 90811 90823 90843 90845 90849 90855 90865 90879 90903 90999 91055 91069 91213 91765 91767 91779 91785 91791 91797 91803 91809 91811 91815 91825 91827 91829 91833 91841 91855 91863 91865 91867 91871 91873 91879 91883 91895 91897 91899 91905 91931 91947 91949 91957 91961 91963 91967 91969 91975 91977 91979 91981 91983 91985 91987 91991 91995 92007 92009 92017 92021 92025 92027 92031 92047 92055 92063 92067 92085 92091 92097 92101 92103 92105 92107 92113 92115 92123 92127 92133 92135 92163 92165 92167 92169 92177 92193 92195 92197 92203 92209 92211 92215 92217 92219 92231 92249 92251 92253 92257 92265 92271 92293 92305 92317 92319 92321 92333 92349 92357 92371 92387 92403 92413 92417 92429 92445 92453 92469 92471 92479 92491 92495 92499 92505 92515 92535 92541 92547 92567 92593 92595 92599 92615 92623 92627 92635 92637 92651 92653 92661 92677 92679 92699 92701 92707 92715 92727 92733 92737 92751 92757 92771 92793 92829 92845 92847 92857 92859 92861 92863 92865 92867 92869 92871 92875 92877 92879 92887 92889 92905 92927 92929 92931 92933 92953 92969 92973 92975 92977 92983 93015 93023 93025 93027 93031 93033 93035 93037 93039 93041 93043 93045 93047 93049 93053 93055 93057 93061 93063 93065 93067 93069 93071 93073) for request_id in "${request_ids[@]}"; do echo idds resume_requests --request_id=${request_id} idds resume_requests --request_id=${request_id} diff --git a/monitor/conf.js b/monitor/conf.js index 66db231f..f5593e45 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus728.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus728.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus728.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus728.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7109.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7109.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7109.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7109.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7109.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7109.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index 72806ea1..d5fa182c 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -160,6 +160,8 @@ def __init__(self, processing_metadata={}): self.output_data = None + self.retries = 0 + @property def internal_id(self): return self.get_metadata_item('internal_id') @@ -204,6 +206,14 @@ def substatus(self, value): if self.processing: self.processing['substatus'] = value + @property + def retries(self): + return self.get_metadata_item('retries', 0) + + @retries.setter + def retries(self, value): + self.add_metadata_item('retries', value) + @property def last_updated_at(self): last_updated_at = self.get_metadata_item('last_updated_at', None) From 30e0514c644a1706bbbf78aa46453afaf27c974b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 26 Aug 2021 15:50:34 +0200 Subject: [PATCH 058/156] new version 0.7.4 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 8e347f2a..c902733c 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 942e0228..75a9b2c0 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.3 - - idds-workflow==0.7.3 \ No newline at end of file + - idds-common==0.7.4 + - idds-workflow==0.7.4 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 8e347f2a..c902733c 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 11382603..eda79039 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.3 - - idds-workflow==0.7.3 \ No newline at end of file + - idds-common==0.7.4 + - idds-workflow==0.7.4 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 8e347f2a..c902733c 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 8d90eabd..11a843e6 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index f32d046c..91cc3272 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.3 - - idds-workflow==0.7.3 \ No newline at end of file + - idds-common==0.7.4 + - idds-workflow==0.7.4 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 8e347f2a..c902733c 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 05a9997c..246df9f5 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.3 - - idds-workflow==0.7.3 - - idds-client==0.7.3 \ No newline at end of file + - idds-common==0.7.4 + - idds-workflow==0.7.4 + - idds-client==0.7.4 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 8e347f2a..c902733c 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/website/version.py b/website/version.py index 8e347f2a..c902733c 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 8e347f2a..c902733c 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.3" +release_version = "0.7.4" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index f344dcf8..249026d9 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.3 \ No newline at end of file + - idds-common==0.7.4 \ No newline at end of file From 7fa19b56fa5efc73ee45703bfe30a4b17d9e89f9 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 8 Sep 2021 18:58:22 +0200 Subject: [PATCH 059/156] fix orm requests --- main/lib/idds/orm/requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/lib/idds/orm/requests.py b/main/lib/idds/orm/requests.py index 72b3f074..7ee01c98 100644 --- a/main/lib/idds/orm/requests.py +++ b/main/lib/idds/orm/requests.py @@ -666,7 +666,7 @@ def update_request(request_id, parameters, session=None): if workflow is not None: workflow.refresh_works() - if 'processing_metadata' not in parameters: + if 'processing_metadata' not in parameters or not parameters['processing_metadata']: parameters['processing_metadata'] = {} parameters['processing_metadata']['workflow_data'] = workflow.metadata From 263d83457647f7dbbffdf1e62ed8c68813687a74 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 8 Sep 2021 18:59:21 +0200 Subject: [PATCH 060/156] update retries to be continous failures --- .../idds/agents/transformer/transformer.py | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index d17fd0d2..536621cf 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -46,6 +46,11 @@ def __init__(self, num_threads=1, poll_time_period=1800, retrieve_bulk_size=10, self.retrieve_bulk_size = int(retrieve_bulk_size) self.message_bulk_size = int(message_bulk_size) + if not hasattr(self, 'retries') or not self.retries: + self.retries = 100 + else: + self.retries = int(self.retries) + self.new_task_queue = Queue() self.new_output_queue = Queue() self.running_task_queue = Queue() @@ -381,12 +386,15 @@ def process_new_transform(self, transform): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - if transform['retries'] > 10: + if transform['retries'] > self.retries: tf_status = TransformStatus.Failed else: tf_status = TransformStatus.Transforming + + wait_times = max(4, transform['retries']) + transform_parameters = {'status': tf_status, - 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4), + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * wait_times), 'retries': transform['retries'] + 1, 'locking': TransformLocking.Idle} ret = {'transform': transform, 'transform_parameters': transform_parameters} @@ -974,6 +982,9 @@ def process_running_transform_real(self, transform): else: next_poll_at = datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_operation_time_period) + # reset retries to 0 when it succeed + transform['retries'] = 0 + transform_parameters = {'status': transform['status'], 'locking': TransformLocking.Idle, 'workload_id': transform['workload_id'], @@ -1027,13 +1038,16 @@ def process_running_transform_message(self, transform, messages): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - if transform['retries'] > 10: + if transform['retries'] > self.retries: tf_status = TransformStatus.Failed else: tf_status = TransformStatus.Transforming + + wait_times = max(4, transform['retries']) + ret = {'transform': transform, 'transform_parameters': {'status': tf_status, - 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4), + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * wait_times), 'locking': TransformLocking.Idle, 'retries': transform['retries'] + 1, 'errors': {'msg': '%s: %s' % (ex, traceback.format_exc())}}} @@ -1054,12 +1068,15 @@ def process_running_transform(self, transform): except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - if transform['retries'] > 10: + if transform['retries'] > self.retries: tf_status = TransformStatus.Failed else: tf_status = TransformStatus.Transforming + + wait_times = max(4, transform['retries']) + transform_parameters = {'status': tf_status, - 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4), + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * wait_times), 'retries': transform['retries'] + 1, 'locking': TransformLocking.Idle} ret = {'transform': transform, 'transform_parameters': transform_parameters} From 60306c1f9993e75aa8015f98687239e2dac1bd73 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 8 Sep 2021 18:59:51 +0200 Subject: [PATCH 061/156] update tests --- main/lib/idds/agents/archive/run_archive.py | 2 +- main/lib/idds/tests/resume_subfinished_data_carousel.sh | 3 +++ main/lib/idds/tests/test_migrate_requests.py | 8 ++++---- main/lib/idds/tests/trigger_release.py | 3 ++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/main/lib/idds/agents/archive/run_archive.py b/main/lib/idds/agents/archive/run_archive.py index 486c4e6f..141e62f0 100644 --- a/main/lib/idds/agents/archive/run_archive.py +++ b/main/lib/idds/agents/archive/run_archive.py @@ -27,7 +27,7 @@ def get_archive_sql(schema): updated_at, next_poll_at, accessed_at, expired_at, errors, request_metadata, processing_metadata FROM {schema}.requests - WHERE status in (3, 5, 9, 17) and created_at < sysdate - interval '7' month + WHERE status in (3, 5, 9, 17) and created_at < sysdate - interval '6' month order by request_id asc) LOOP --- archive records diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh index 39e55bc6..75706e42 100644 --- a/main/lib/idds/tests/resume_subfinished_data_carousel.sh +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -6,6 +6,9 @@ request_ids=(75039 77281 77283 77285 77287 83271 83483 77279 77367 77847 77947 7 request_ids=(7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 16539 16541 16543 16545 16547 16549 16825 17615 19289 19319 19321 19397 75039 77279 77281 77283 77285 77287 77947 80109 82731 83069 83925 83933 83943 83945 84081 84083 84091 84125 84145 84181 84199 84569 87199 87661 87715 88841 89579 89891 89917 89937 89939 89941 89949 89955 89997 90165 90471 90795 90855 91069 91765 91779 91797 91803 91841 91865 91867 91879 91949 91961 91969 91975 91981 91983 91985 92009 92017 92021 92025 92027 92031 92055 92063 92097 92105 92107 92113 92133 92163 92165 92169 92197 92203 92209 92211 92231 92265 92305 92319 92387 92417 92445 92453 92495 92499 92535 92595 92623 92757 92861 92865 92871 92879 92889 92975 92983 93023 93031 93033 93049) +request_ids=(75039 77279 77281 77283 77285 77287 80109 83069 83943 83945 84125 84145 84181 84199 84569 89917 89941 89949 90471 90795 90823 90845 90855 91949 92929 92931 92973 92975 92977 92983 93035 93049 93053 93057 93087 93089 93093 93145 93291 93391 93447 93537 93555 93623 93715 93733 93819 93901 93957 93961 93989 94057 94067 94069 94275 94431 94471 94473 94567 94581 94615 94621 94623 94625 94629 94633 94641 94659 94707 94713 94721 94723 94725 94727 94731 94737 94739 94741 94743 94745 94747 94749 94751 94753 94757 94761 94763 94765) + + for request_id in "${request_ids[@]}"; do echo idds resume_requests --request_id=${request_id} idds resume_requests --request_id=${request_id} diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index 94da858b..8814be3d 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -29,16 +29,16 @@ def migrate(): # atlas atlas_host = 'https://aipanda181.cern.ch:443/idds' # noqa F841 - cm1 = ClientManager(host=doma_host) + cm1 = ClientManager(host=atlas_host) # reqs = cm1.get_requests(request_id=290) - old_request_id = 72533 + old_request_id = 72521 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 # for old_request_id in [200]: # noqa E115 - for old_request_id in [183]: # noqa E115 # doma 183 + for old_request_id in [72521]: # noqa E115 # doma 183 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=dev_host) + cm2 = ClientManager(host=atlas_host) for req in reqs: req = convert_old_req_2_workflow_req(req) workflow = req['request_metadata']['workflow'] diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 91573e77..70e8dc16 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -11,7 +11,8 @@ from idds.orm.contents import get_input_contents # noqa F401 -contents = get_contents(request_id=202, status=ContentStatus.Available) +request_id = 221 +contents = get_contents(request_id=request_id, status=ContentStatus.Available) ret_contents = {} for content in contents: if content['content_relation_type'] == ContentRelationType.Output: # InputDependency From 062c24aa37c8c746d4d5e55a5e58f6fde0291ead Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 8 Sep 2021 19:24:43 +0200 Subject: [PATCH 062/156] improve message inde --- main/lib/idds/orm/messages.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/main/lib/idds/orm/messages.py b/main/lib/idds/orm/messages.py index ba8f46c1..b5b9eb93 100644 --- a/main/lib/idds/orm/messages.py +++ b/main/lib/idds/orm/messages.py @@ -126,7 +126,15 @@ def retrieve_messages(bulk_size=1000, msg_type=None, status=None, source=None, messages = [] try: query = session.query(models.Message) - query = query.with_hint(models.Message, "INDEX(MESSAGES MESSAGES_TYPE_ST_IDX)", 'oracle') + if request_id is not None: + query = query.with_hint(models.Message, "INDEX(MESSAGES MESSAGES_TYPE_ST_IDX)", 'oracle') + elif transform_id: + query = query.with_hint(models.Message, "INDEX(MESSAGES MESSAGES_TYPE_ST_TF_IDX)", 'oracle') + elif processing_id is not None: + query = query.with_hint(models.Message, "INDEX(MESSAGES MESSAGES_TYPE_ST_PR_IDX)", 'oracle') + else: + query = query.with_hint(models.Message, "INDEX(MESSAGES MESSAGES_TYPE_ST_IDX)", 'oracle') + if msg_type is not None: query = query.filter_by(msg_type=msg_type) if status is not None: From 75d6633c9a8af308cc79cb372936227b5164664d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 9 Sep 2021 11:23:51 +0200 Subject: [PATCH 063/156] fix exception for old requests --- main/lib/idds/agents/carrier/carrier.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/main/lib/idds/agents/carrier/carrier.py b/main/lib/idds/agents/carrier/carrier.py index a72d606a..c88b0430 100644 --- a/main/lib/idds/agents/carrier/carrier.py +++ b/main/lib/idds/agents/carrier/carrier.py @@ -305,6 +305,9 @@ def process_running_processing(self, processing): # work = transform['transform_metadata']['work'] # work = processing['processing_metadata']['work'] # work.set_agent_attributes(self.agent_attributes) + if 'processing' not in processing['processing_metadata']: + raise exceptions.ProcessFormatNotSupported + proc = processing['processing_metadata']['processing'] work = proc.work work.set_agent_attributes(self.agent_attributes, processing) @@ -381,6 +384,16 @@ def process_running_processing(self, processing): ret = {'processing_update': processing_update, 'content_updates': content_updates, 'new_contents': new_contents} + + except exceptions.ProcessFormatNotSupported as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + processing_update = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Failed, + 'locking': ProcessingLocking.Idle, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4)}} + ret = {'processing_update': processing_update, + 'content_updates': []} except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) From 07d71166d7901499747feba22b1d1ea132a9cd0a Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 9 Sep 2021 11:24:07 +0200 Subject: [PATCH 064/156] new version 0.7.5 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/exceptions.py | 10 ++++++++++ common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/etc/sql/oracle_11.sql | 2 ++ main/etc/sql/oracle_19.sql | 2 ++ .../idds/tests/resume_subfinished_data_carousel.sh | 1 + main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/conf.js | 12 ++++++------ monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 18 files changed, 39 insertions(+), 24 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index c902733c..e4a68b6f 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 75a9b2c0..9e48f657 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.4 - - idds-workflow==0.7.4 \ No newline at end of file + - idds-common==0.7.5 + - idds-workflow==0.7.5 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index c902733c..e4a68b6f 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index eda79039..f9194802 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.4 - - idds-workflow==0.7.4 \ No newline at end of file + - idds-common==0.7.5 + - idds-workflow==0.7.5 \ No newline at end of file diff --git a/common/lib/idds/common/exceptions.py b/common/lib/idds/common/exceptions.py index dc76a6cf..cc0a09fa 100644 --- a/common/lib/idds/common/exceptions.py +++ b/common/lib/idds/common/exceptions.py @@ -194,6 +194,16 @@ def __init__(self, *args, **kwargs): self.error_code = 403 +class ProcessFormatNotSupported(IDDSException): + """ + ProcessFormatNotSupported + """ + def __init__(self, *args, **kwargs): + super(ProcessFormatNotSupported, self).__init__(*args, **kwargs) + self._message = "Process format not support." + self.error_code = 404 + + class AgentException(IDDSException): """ BrokerException diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index c902733c..e4a68b6f 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 11a843e6..3b255fe8 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 91cc3272..24ab1298 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.4 - - idds-workflow==0.7.4 \ No newline at end of file + - idds-common==0.7.5 + - idds-workflow==0.7.5 \ No newline at end of file diff --git a/main/etc/sql/oracle_11.sql b/main/etc/sql/oracle_11.sql index 83c1188a..73ed8e0c 100644 --- a/main/etc/sql/oracle_11.sql +++ b/main/etc/sql/oracle_11.sql @@ -418,6 +418,8 @@ alter table messages add destination NUMBER(2); alter table messages add processing_id NUMBER(12); CREATE INDEX MESSAGES_TYPE_ST_IDX ON MESSAGES (msg_type, status, destination, request_id); +CREATE INDEX MESSAGES_TYPE_ST_TF_IDX ON MESSAGES (msg_type, status, destination, transform_id); +CREATE INDEX MESSAGES_TYPE_ST_PR_IDX ON MESSAGES (msg_type, status, destination, processing_id); --- health diff --git a/main/etc/sql/oracle_19.sql b/main/etc/sql/oracle_19.sql index ba3dafeb..9335f7a5 100644 --- a/main/etc/sql/oracle_19.sql +++ b/main/etc/sql/oracle_19.sql @@ -324,6 +324,8 @@ alter table messages add destination NUMBER(2); alter table messages add processing_id NUMBER(12); CREATE INDEX MESSAGES_TYPE_ST_IDX ON MESSAGES (msg_type, status, destination, request_id); +CREATE INDEX MESSAGES_TYPE_ST_TF_IDX ON MESSAGES (msg_type, status, destination, transform_id); +CREATE INDEX MESSAGES_TYPE_ST_PR_IDX ON MESSAGES (msg_type, status, destination, processing_id); --- health CREATE SEQUENCE HEALTH_ID_SEQ MINVALUE 1 INCREMENT BY 1 START WITH 1 NOCACHE ORDER NOCYCLE GLOBAL; diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh index 75706e42..96a77795 100644 --- a/main/lib/idds/tests/resume_subfinished_data_carousel.sh +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -8,6 +8,7 @@ request_ids=(7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 165 request_ids=(75039 77279 77281 77283 77285 77287 80109 83069 83943 83945 84125 84145 84181 84199 84569 89917 89941 89949 90471 90795 90823 90845 90855 91949 92929 92931 92973 92975 92977 92983 93035 93049 93053 93057 93087 93089 93093 93145 93291 93391 93447 93537 93555 93623 93715 93733 93819 93901 93957 93961 93989 94057 94067 94069 94275 94431 94471 94473 94567 94581 94615 94621 94623 94625 94629 94633 94641 94659 94707 94713 94721 94723 94725 94727 94731 94737 94739 94741 94743 94745 94747 94749 94751 94753 94757 94761 94763 94765) +request_ids=(75039 77279 77281 77285 77287 83069 83943 83945 84125 84145 84199 84569 90845 90855 94057 94067 94069 94275 94797 94803 94805 94807 94809 94811 94813 94815 94817 94819 94821 94823 94825 94827 94829 94831 94833 94835 94837 94839 94841 94843 94845 94847 94849 94851 94853 94855 94857 94859 94861 94863 94865 94867 94869 94871 94873 94875 94877 94879 94881 94883 94887 94889 94891 94893 94895 94897 94901 94903 94905 94907 94911 94913 94915 94917 94919 94921 94923 94925 94927 94931 94933 94935 94937 94939 94941 94943 94945 94947 94949 94951 94953 94955 94957 94959 94961 94963 94965 94967 94969 94971 94973 94975 94977 94979 94981 94983 94985 94987 94989 94991 94993 94997 94999 95001 95003 95005 95007 95009 95011 95013 95015 95017 95019 95021 95025 95027 95029 95031 95033 95035 95037 95039 95041 95043 95045 95047 95049 95051 95055 95057 95059 95063 95065 95067 95069 95071 95073 95075 95077 95079 95081 95083 95085 95087 95089 95091 95093 95095 95097 95099 95101 95103 95105 95107 95109 95111 95113 95115 95117 95119 95121 95123 95125 95127 96417) for request_id in "${request_ids[@]}"; do echo idds resume_requests --request_id=${request_id} diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index c902733c..e4a68b6f 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 246df9f5..f565dcba 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.4 - - idds-workflow==0.7.4 - - idds-client==0.7.4 \ No newline at end of file + - idds-common==0.7.5 + - idds-workflow==0.7.5 + - idds-client==0.7.5 \ No newline at end of file diff --git a/monitor/conf.js b/monitor/conf.js index f5593e45..e559ac83 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus7109.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus7109.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus7109.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus7109.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus7109.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus7109.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus739.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus739.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus739.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/version.py b/monitor/version.py index c902733c..e4a68b6f 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/website/version.py b/website/version.py index c902733c..e4a68b6f 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index c902733c..e4a68b6f 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.4" +release_version = "0.7.5" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 249026d9..5d909319 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.4 \ No newline at end of file + - idds-common==0.7.5 \ No newline at end of file From 6cc8f93eeb4c7a2bdd191915005bcb341c90af84 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 13 Sep 2021 13:41:06 +0200 Subject: [PATCH 065/156] fix collection messages --- .../idds/agents/transformer/transformer.py | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 536621cf..ac461e7a 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -903,73 +903,97 @@ def process_running_transform_real(self, transform): transform['status'] = TransformStatus.Finished msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) + for coll in input_collections: + coll.status = CollectionStatus.Closed + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') + msgs.append(msg) for coll in output_collections: coll.status = CollectionStatus.Closed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='output') msgs.append(msg) for coll in log_collections: coll.status = CollectionStatus.Closed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='log') msgs.append(msg) elif work.is_subfinished(): transform['status'] = TransformStatus.SubFinished msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) + for coll in in_collections: + coll.status = CollectionStatus.SubClosed + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') + msgs.append(msg) for coll in output_collections: coll.status = CollectionStatus.SubClosed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='output') msgs.append(msg) for coll in log_collections: coll.status = CollectionStatus.SubClosed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='log') msgs.append(msg) elif work.is_failed(): transform['status'] = TransformStatus.Failed msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) + for coll in input_collections: + coll.status = CollectionStatus.Failed + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') + msgs.append(msg) for coll in output_collections: coll.status = CollectionStatus.Failed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='output') msgs.append(msg) for coll in log_collections: coll.status = CollectionStatus.Failed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='log') msgs.append(msg) elif work.is_expired(): transform['status'] = TransformStatus.Expired msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) + for coll in input_collections: + coll.status = CollectionStatus.SubClosed + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') + msgs.append(msg) for coll in output_collections: coll.status = CollectionStatus.SubClosed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='output') msgs.append(msg) for coll in log_collections: coll.status = CollectionStatus.SubClosed - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='log') msgs.append(msg) elif work.is_cancelled(): transform['status'] = TransformStatus.Cancelled msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) + for coll in input_collections: + coll.status = CollectionStatus.Cancelled + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') + msgs.append(msg) for coll in output_collections: coll.status = CollectionStatus.Cancelled - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='output') msgs.append(msg) for coll in log_collections: coll.status = CollectionStatus.Cancelled - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='log') msgs.append(msg) elif work.is_suspended(): transform['status'] = TransformStatus.Suspended msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) + for coll in input_collections: + coll.status = CollectionStatus.Suspended + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') + msgs.append(msg) for coll in output_collections: coll.status = CollectionStatus.Suspended - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='output') msgs.append(msg) for coll in log_collections: coll.status = CollectionStatus.Suspended - msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection') + msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='log') msgs.append(msg) else: transform['status'] = TransformStatus.Transforming From 89979b64649f9107da00c029a5487e7e388cdf3b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 13 Sep 2021 13:47:06 +0200 Subject: [PATCH 066/156] data carousel input and output should the same name --- main/lib/idds/rest/v1/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/lib/idds/rest/v1/utils.py b/main/lib/idds/rest/v1/utils.py index 1d1b9d68..52fd6572 100644 --- a/main/lib/idds/rest/v1/utils.py +++ b/main/lib/idds/rest/v1/utils.py @@ -37,7 +37,7 @@ def convert_stagein_request_metadata_to_workflow(scope, name, workload_id, reque exec_type='local', sandbox=None, primary_input_collection={'scope': scope, 'name': name}, other_input_collections=None, - output_collections={'scope': scope, 'name': name + '.idds.stagein'}, + output_collections={'scope': scope, 'name': name}, log_collections=None, logger=None, max_waiting_time=request_metadata.get('max_waiting_time', 3600 * 7 * 24), From bdda1557358c90dcfe6f44c86520c0b73ce2c366 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 13 Sep 2021 13:56:59 +0200 Subject: [PATCH 067/156] fix reactive collections --- main/lib/idds/agents/transformer/transformer.py | 16 ++++++++++++++-- .../tests/resume_subfinished_data_carousel.sh | 2 +- main/lib/idds/tests/run_sql.py | 2 ++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index ac461e7a..98fbffde 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -563,13 +563,18 @@ def generate_message(self, transform, work=None, collection=None, files=None, ms 'error': work.get_terminated_msg()} num_msg_content = 1 elif msg_type == 'collection': + # fix for old requests + coll_name = collection.name + if coll_name.endswith(".idds.stagein"): + coll_name = coll_name.replace(".idds.stagein", "") + i_msg_type, i_msg_type_str = self.get_message_type(transform['transform_type'], input_type='collection') msg_content = {'msg_type': i_msg_type_str, 'request_id': request_id, 'workload_id': workload_id, 'relation_type': relation_type, 'collections': [{'scope': collection.scope, - 'name': collection.name, + 'name': coll_name, 'status': collection.status.name}], 'output': work.get_output_data(), 'error': work.get_terminated_msg()} @@ -890,6 +895,13 @@ def process_running_transform_real(self, transform): work.toresume = True to_resume_transform = True reactivated_contents = self.reactive_contents(registered_input_output_maps) + # reactive collections + for coll in input_collections: + coll.status = CollectionStatus.Open + for coll in output_collections: + coll.status = CollectionStatus.Open + for coll in log_collections: + coll.status = CollectionStatus.Open elif transform['status'] in [TransformStatus.ToExpire]: transform['status'] = TransformStatus.Expiring work.toexpire = True @@ -919,7 +931,7 @@ def process_running_transform_real(self, transform): transform['status'] = TransformStatus.SubFinished msg = self.generate_message(transform, work=work, msg_type='work') msgs.append(msg) - for coll in in_collections: + for coll in input_collections: coll.status = CollectionStatus.SubClosed msg = self.generate_message(transform, work=work, collection=coll, msg_type='collection', relation_type='input') msgs.append(msg) diff --git a/main/lib/idds/tests/resume_subfinished_data_carousel.sh b/main/lib/idds/tests/resume_subfinished_data_carousel.sh index 96a77795..9b253366 100644 --- a/main/lib/idds/tests/resume_subfinished_data_carousel.sh +++ b/main/lib/idds/tests/resume_subfinished_data_carousel.sh @@ -8,7 +8,7 @@ request_ids=(7041 7241 11673 16523 16525 16527 16529 16531 16533 16535 16537 165 request_ids=(75039 77279 77281 77283 77285 77287 80109 83069 83943 83945 84125 84145 84181 84199 84569 89917 89941 89949 90471 90795 90823 90845 90855 91949 92929 92931 92973 92975 92977 92983 93035 93049 93053 93057 93087 93089 93093 93145 93291 93391 93447 93537 93555 93623 93715 93733 93819 93901 93957 93961 93989 94057 94067 94069 94275 94431 94471 94473 94567 94581 94615 94621 94623 94625 94629 94633 94641 94659 94707 94713 94721 94723 94725 94727 94731 94737 94739 94741 94743 94745 94747 94749 94751 94753 94757 94761 94763 94765) -request_ids=(75039 77279 77281 77285 77287 83069 83943 83945 84125 84145 84199 84569 90845 90855 94057 94067 94069 94275 94797 94803 94805 94807 94809 94811 94813 94815 94817 94819 94821 94823 94825 94827 94829 94831 94833 94835 94837 94839 94841 94843 94845 94847 94849 94851 94853 94855 94857 94859 94861 94863 94865 94867 94869 94871 94873 94875 94877 94879 94881 94883 94887 94889 94891 94893 94895 94897 94901 94903 94905 94907 94911 94913 94915 94917 94919 94921 94923 94925 94927 94931 94933 94935 94937 94939 94941 94943 94945 94947 94949 94951 94953 94955 94957 94959 94961 94963 94965 94967 94969 94971 94973 94975 94977 94979 94981 94983 94985 94987 94989 94991 94993 94997 94999 95001 95003 95005 95007 95009 95011 95013 95015 95017 95019 95021 95025 95027 95029 95031 95033 95035 95037 95039 95041 95043 95045 95047 95049 95051 95055 95057 95059 95063 95065 95067 95069 95071 95073 95075 95077 95079 95081 95083 95085 95087 95089 95091 95093 95095 95097 95099 95101 95103 95105 95107 95109 95111 95113 95115 95117 95119 95121 95123 95125 95127 96417) +request_ids=(98471 98481 98487 98507 98515 98527 98525 98547 98553 98559 98573 98623 98647 98667 98685 98689 98721 98729 98739 98741 98779 98781 98835 98849 98853 98883 98887 98911 98919 98629 98631 98645 98723 98749 98757 98803 98815 98821 98827 98831 98845 98861 98873 98893 98905 98609 98611 98867 98871 98869 98875 98877 98889 98891 98901 98907 98915 98917 98475 98493 98495 98539 98551 98563 98591 98599 98601 98605 98607 98615 98649 98655 98659 98713 98715 98745 98747 98755 98765 98773 98785 98791 98799 98807 98811 98809 98817 98819 98837 98839 98841 98847 98865 98727 98731 98737 98759 98763 98769 98771 98775 98783 98795 98825 98833 98851 98855 98857 98859 98899 98897 98921 98469 98477 98483 98491 98511 98513 98521 98529 98541 98545 98555 98567 98571 98579 98583 98581 98585 98593 98597 98603 98613 98617 98619 98633 98637 98635 98641 98643 98653 98657 98663 98669 98671 98675 98677 98679 98683 98691 98687 98693 98701 98697 98695 98703 98705 98707 98711 98717 98725 98473 98501 98499 98519 98535 98533 98517 98537 98549 98561 98569 98479 98485 98489 98505 98509 98523 98531 98543 98557 98565 98575 98577 98587 98595 98651 98665 98681 98699 98709 98719 98733 98735 98743 98753 98761 98767 98777 98797 98801 98805 98885 98895 98903 98913 98503 98497 98589 98621 98625 98627 98639 98661 98673 98751 98787 98789 98793 98813 98823 98829 98843 98863 98879 98881 98909 32325 32205 49959 53567 49961 42629 87581 87753 87689 87693 98923 98925) for request_id in "${request_ids[@]}"; do echo idds resume_requests --request_id=${request_id} diff --git a/main/lib/idds/tests/run_sql.py b/main/lib/idds/tests/run_sql.py index 0b85a328..cde19d19 100644 --- a/main/lib/idds/tests/run_sql.py +++ b/main/lib/idds/tests/run_sql.py @@ -25,6 +25,8 @@ def get_subfinished_requests(db_pool): req_ids = [] # sql = """select request_id from atlas_IDDS.requests where status in (4,5) and scope!='hpo'""" sql = """select request_id from atlas_IDDS.requests where scope!='hpo' and ( status in (4,5) or request_id in (select request_id from atlas_idds.transforms where status in (4, 5) and transform_type=2)) order by request_id""" + sql = """select request_id from atlas_idds.collections where status=4 and total_files > processed_files order by request_id asc""" + cursor = connection.cursor() cursor.execute(sql) rows = cursor.fetchall() From ef785918786a62c2f4da545d03aeaa1c8bd9e65f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 13 Sep 2021 16:40:57 +0200 Subject: [PATCH 068/156] optimize messages for data carousel --- atlas/lib/idds/atlas/workflow/atlasstageinwork.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py index 64b2fd27..00eb9e62 100644 --- a/atlas/lib/idds/atlas/workflow/atlasstageinwork.py +++ b/atlas/lib/idds/atlas/workflow/atlasstageinwork.py @@ -200,8 +200,8 @@ def get_new_input_output_maps(self, mapped_input_output_maps={}): for ip in new_inputs: self.num_mapped_inputs += 1 out_ip = copy.deepcopy(ip) - ip['status'] = ContentStatus.Available - ip['substatus'] = ContentStatus.Available + ip['status'] = ContentStatus.New + ip['substatus'] = ContentStatus.New out_ip['coll_id'] = self.collections[self.output_collections[0]].coll_id new_input_output_maps[next_key] = {'inputs': [ip], 'outputs': [out_ip], @@ -320,8 +320,9 @@ def poll_processing_updates(self, processing, input_output_maps): updated_contents = [] content_substatus = {'finished': 0, 'unfinished': 0} for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] outputs = input_output_maps[map_id]['outputs'] - for content in outputs: + for content in inputs + outputs: key = '%s:%s' % (content['scope'], content['name']) if key in rep_status: if content['substatus'] != rep_status[key]: @@ -359,8 +360,9 @@ def poll_processing_updates(self, processing, input_output_maps): 'parameters': {'status': ProcessingStatus.SubFinished}} elif self.toforcefinish: for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] outputs = input_output_maps[map_id]['outputs'] - for content in outputs: + for content in inputs + outputs: if content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable]: updated_content = {'content_id': content['content_id'], 'substatus': ContentStatus.FakeAvailable} From 03e8965a5e629e6df7078fcf8b3e27e33f18603a Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 14 Sep 2021 16:19:39 +0200 Subject: [PATCH 069/156] fix sync works --- main/etc/sql/oracle_19.sql | 3 +++ main/lib/idds/agents/clerk/clerk.py | 2 +- main/lib/idds/tests/core_tests.py | 2 +- workflow/lib/idds/workflow/workflow.py | 4 ++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/main/etc/sql/oracle_19.sql b/main/etc/sql/oracle_19.sql index 9335f7a5..7c0701ce 100644 --- a/main/etc/sql/oracle_19.sql +++ b/main/etc/sql/oracle_19.sql @@ -40,6 +40,8 @@ CREATE TABLE REQUESTS name VARCHAR2(255) constraint REQ_NAME_NN NOT NULL, requester VARCHAR2(20), request_type NUMBER(2) constraint REQ_DATATYPE_NN NOT NULL, + username VARCHAR2(20), + userdn VARCHAR2(200), transform_tag VARCHAR2(10), workload_id NUMBER(10), priority NUMBER(7), @@ -66,6 +68,7 @@ CREATE INDEX REQUESTS_SCOPE_NAME_IDX ON REQUESTS (name, scope, workload_id) LOCA --- drop index REQUESTS_STATUS_PRIORITY_IDX CREATE INDEX REQUESTS_STATUS_PRIORITY_IDX ON REQUESTS (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; +alter table REQUESTS modify (min_id NUMBER(7) default 0) --- workprogress CREATE SEQUENCE WORKPROGRESS_ID_SEQ MINVALUE 1 INCREMENT BY 1 ORDER NOCACHE NOCYCLE GLOBAL; diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index cb2721e3..c64fb074 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -288,7 +288,7 @@ def process_running_request_real(self, req): self.logger.debug("Processing request(%s): new transforms: %s" % (req['request_id'], str(new_transforms))) # current works - works = wf.get_current_works() + works = wf.get_all_works() # print(works) for work in works: # print(work.get_work_id()) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 73f4a02c..cd3fdd48 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,7 +102,7 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=161, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=256, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index a89f2597..c979a175 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -1124,6 +1124,10 @@ def sync_works(self): self.refresh_works() + for k in self.works: + work = self.works[k] + self.log_debug("work %s is_terminated(%s:%s)" % (work.get_internal_id(), work.is_terminated(), work.get_status())) + for work in [self.works[k] for k in self.new_to_run_works]: if work.transforming: self.new_to_run_works.remove(work.get_internal_id()) From 5435b753555c64cbca6b36015f8227c087a93458 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 14 Sep 2021 16:25:43 +0200 Subject: [PATCH 070/156] fix sync works --- workflow/lib/idds/workflow/workflow.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index c979a175..bc8c36b9 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -1038,6 +1038,15 @@ def get_current_works(self): self.sync_works() return [self.works[k] for k in self.current_running_works] + def get_all_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + return [self.works[k] for k in self.works] + def get_primary_initial_collection(self): """ *** Function called by Clerk agent. From b828293b5e38ef88611cc8ed13a7b35b1d317fde Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 14 Sep 2021 16:41:26 +0200 Subject: [PATCH 071/156] avoid duplicated expiring messages --- workflow/lib/idds/workflow/workflow.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index bc8c36b9..afa50bdb 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -1269,6 +1269,9 @@ def is_failed(self): return self.is_terminated() and (self.num_failed_works > 0) and (self.num_cancelled_works == 0) and (self.num_suspended_works == 0) and (self.num_expired_works == 0) def is_to_expire(self, expired_at=None, pending_time=None, request_id=None): + if self.expired: + # it's already expired. avoid sending duplicated messages again and again. + return False if expired_at: if type(expired_at) in [str]: expired_at = str_to_date(expired_at) From f6038ecb43e1d8dcbf0596a963664bcebfaac7d7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 14 Sep 2021 16:41:40 +0200 Subject: [PATCH 072/156] add username to requests table --- main/etc/sql/oracle_11.sql | 5 +++++ main/etc/sql/oracle_19.sql | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/main/etc/sql/oracle_11.sql b/main/etc/sql/oracle_11.sql index 73ed8e0c..a6909d13 100644 --- a/main/etc/sql/oracle_11.sql +++ b/main/etc/sql/oracle_11.sql @@ -40,6 +40,8 @@ CREATE TABLE REQUESTS name VARCHAR2(255) constraint REQ_NAME_NN NOT NULL, requester VARCHAR2(20), request_type NUMBER(2) constraint REQ_DATATYPE_NN NOT NULL, + username VARCHAR2(20) default null, + userdn VARCHAR2(200) default null, transform_tag VARCHAR2(10), workload_id NUMBER(10), priority NUMBER(7), @@ -75,6 +77,9 @@ CREATE INDEX REQUESTS_SCOPE_NAME_IDX ON REQUESTS (name, scope, workload_id) LOCA --- drop index REQUESTS_STATUS_PRIORITY_IDX CREATE INDEX REQUESTS_STATUS_PRIORITY_IDX ON REQUESTS (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; +--alter table REQUESTS add (username VARCHAR2(20) default null); +--alter table REQUESTS add (userdn VARCHAR2(200) default null); + --- workprogress CREATE SEQUENCE WORKPROGRESS_ID_SEQ MINVALUE 1 INCREMENT BY 1 ORDER NOCACHE; diff --git a/main/etc/sql/oracle_19.sql b/main/etc/sql/oracle_19.sql index 7c0701ce..5b55d0f2 100644 --- a/main/etc/sql/oracle_19.sql +++ b/main/etc/sql/oracle_19.sql @@ -40,8 +40,8 @@ CREATE TABLE REQUESTS name VARCHAR2(255) constraint REQ_NAME_NN NOT NULL, requester VARCHAR2(20), request_type NUMBER(2) constraint REQ_DATATYPE_NN NOT NULL, - username VARCHAR2(20), - userdn VARCHAR2(200), + username VARCHAR2(20) default null, + userdn VARCHAR2(200) default null, transform_tag VARCHAR2(10), workload_id NUMBER(10), priority NUMBER(7), @@ -68,7 +68,8 @@ CREATE INDEX REQUESTS_SCOPE_NAME_IDX ON REQUESTS (name, scope, workload_id) LOCA --- drop index REQUESTS_STATUS_PRIORITY_IDX CREATE INDEX REQUESTS_STATUS_PRIORITY_IDX ON REQUESTS (status, priority, request_id, locking, updated_at, next_poll_at, created_at) LOCAL COMPRESS 1; -alter table REQUESTS modify (min_id NUMBER(7) default 0) +-- alter table REQUESTS add (username VARCHAR2(20) default null); +-- alter table REQUESTS add (userdn VARCHAR2(200) default null); --- workprogress CREATE SEQUENCE WORKPROGRESS_ID_SEQ MINVALUE 1 INCREMENT BY 1 ORDER NOCACHE NOCYCLE GLOBAL; From 0078253695a34a49db12200bf0a789fe905f40ed Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 14 Sep 2021 17:21:04 +0200 Subject: [PATCH 073/156] add username and userdn --- client/lib/idds/client/clientmanager.py | 2 ++ main/lib/idds/core/requests.py | 8 ++++++-- main/lib/idds/orm/base/models.py | 2 ++ main/lib/idds/orm/requests.py | 8 ++++++-- main/lib/idds/tests/test_datacarousel.py | 2 ++ workflow/lib/idds/workflow/workflow.py | 2 ++ 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/client/lib/idds/client/clientmanager.py b/client/lib/idds/client/clientmanager.py index 9579befd..284e0b8b 100644 --- a/client/lib/idds/client/clientmanager.py +++ b/client/lib/idds/client/clientmanager.py @@ -49,6 +49,8 @@ def submit(self, workflow): 'name': workflow.name, 'requester': 'panda', 'request_type': RequestType.Workflow, + 'username': workflow.username, + 'userdn': workflow.userdn, 'transform_tag': 'workflow', 'status': RequestStatus.New, 'priority': 0, diff --git a/main/lib/idds/core/requests.py b/main/lib/idds/core/requests.py index a060beba..0a07b305 100644 --- a/main/lib/idds/core/requests.py +++ b/main/lib/idds/core/requests.py @@ -26,7 +26,8 @@ from idds.core import messages as core_messages -def create_request(scope=None, name=None, requester=None, request_type=None, transform_tag=None, +def create_request(scope=None, name=None, requester=None, request_type=None, + username=None, userdn=None, transform_tag=None, status=RequestStatus.New, locking=RequestLocking.Idle, priority=0, lifetime=None, workload_id=None, request_metadata=None, processing_metadata=None): @@ -53,6 +54,7 @@ def create_request(scope=None, name=None, requester=None, request_type=None, tra # request_metadata = convert_request_metadata_to_workflow(scope, name, workload_id, request_type, request_metadata) kwargs = {'scope': scope, 'name': name, 'requester': requester, 'request_type': request_type, + 'username': username, 'userdn': userdn, 'transform_tag': transform_tag, 'status': status, 'locking': locking, 'priority': priority, 'lifetime': lifetime, 'workload_id': workload_id, 'request_metadata': request_metadata, 'processing_metadata': processing_metadata} @@ -60,7 +62,8 @@ def create_request(scope=None, name=None, requester=None, request_type=None, tra @transactional_session -def add_request(scope=None, name=None, requester=None, request_type=None, transform_tag=None, +def add_request(scope=None, name=None, requester=None, request_type=None, + username=None, userdn=None, transform_tag=None, status=RequestStatus.New, locking=RequestLocking.Idle, priority=0, lifetime=None, workload_id=None, request_metadata=None, processing_metadata=None, session=None): @@ -87,6 +90,7 @@ def add_request(scope=None, name=None, requester=None, request_type=None, transf # request_metadata = convert_request_metadata_to_workflow(scope, name, workload_id, request_type, request_metadata) kwargs = {'scope': scope, 'name': name, 'requester': requester, 'request_type': request_type, + 'username': username, 'userdn': userdn, 'transform_tag': transform_tag, 'status': status, 'locking': locking, 'priority': priority, 'lifetime': lifetime, 'workload_id': workload_id, 'request_metadata': request_metadata, 'processing_metadata': processing_metadata, diff --git a/main/lib/idds/orm/base/models.py b/main/lib/idds/orm/base/models.py index fc790bff..657af50c 100644 --- a/main/lib/idds/orm/base/models.py +++ b/main/lib/idds/orm/base/models.py @@ -135,6 +135,8 @@ class Request(BASE, ModelBase): name = Column(String(NAME_LENGTH)) requester = Column(String(20)) request_type = Column(EnumWithValue(RequestType)) + username = Column(String(20)) + userdn = Column(String(200)) transform_tag = Column(String(20)) workload_id = Column(Integer()) priority = Column(Integer()) diff --git a/main/lib/idds/orm/requests.py b/main/lib/idds/orm/requests.py index 7ee01c98..669c9920 100644 --- a/main/lib/idds/orm/requests.py +++ b/main/lib/idds/orm/requests.py @@ -27,7 +27,8 @@ from idds.orm.base import models -def create_request(scope=None, name=None, requester=None, request_type=None, transform_tag=None, +def create_request(scope=None, name=None, requester=None, request_type=None, + username=None, userdn=None, transform_tag=None, status=RequestStatus.New, locking=RequestLocking.Idle, priority=0, lifetime=None, workload_id=None, request_metadata=None, processing_metadata=None): @@ -74,6 +75,7 @@ def create_request(scope=None, name=None, requester=None, request_type=None, tra expired_at = None new_request = models.Request(scope=scope, name=name, requester=requester, request_type=request_type, + username=username, userdn=userdn, transform_tag=transform_tag, status=status, locking=locking, priority=priority, workload_id=workload_id, expired_at=expired_at, @@ -82,7 +84,8 @@ def create_request(scope=None, name=None, requester=None, request_type=None, tra @transactional_session -def add_request(scope=None, name=None, requester=None, request_type=None, transform_tag=None, +def add_request(scope=None, name=None, requester=None, request_type=None, + username=None, userdn=None, transform_tag=None, status=RequestStatus.New, locking=RequestLocking.Idle, priority=0, lifetime=None, workload_id=None, request_metadata=None, processing_metadata=None, session=None): @@ -110,6 +113,7 @@ def add_request(scope=None, name=None, requester=None, request_type=None, transf try: new_request = create_request(scope=scope, name=name, requester=requester, request_type=request_type, + username=username, userdn=userdn, transform_tag=transform_tag, status=status, locking=locking, priority=priority, workload_id=workload_id, lifetime=lifetime, request_metadata=request_metadata, processing_metadata=processing_metadata) diff --git a/main/lib/idds/tests/test_datacarousel.py b/main/lib/idds/tests/test_datacarousel.py index fae340e7..3b3354a0 100644 --- a/main/lib/idds/tests/test_datacarousel.py +++ b/main/lib/idds/tests/test_datacarousel.py @@ -129,6 +129,8 @@ def pre_check(req): # props = get_example_active_learning_request() workflow = get_workflow() + workflow.username = 'abdc' + workflow.userdn = '/DC=abc/DC=def' # props = pre_check(props) # print(props) diff --git a/workflow/lib/idds/workflow/workflow.py b/workflow/lib/idds/workflow/workflow.py index afa50bdb..1dbcf03f 100644 --- a/workflow/lib/idds/workflow/workflow.py +++ b/workflow/lib/idds/workflow/workflow.py @@ -578,6 +578,8 @@ def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None # user defined Condition class self.user_defined_conditions = {} + self.username = None + self.userdn = None self.proxy = None """ From 70a696f0756006a8da4aa62dcf60dec891f5a4cd Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 14 Sep 2021 17:21:35 +0200 Subject: [PATCH 074/156] new version 0.7.6 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/conf.js | 12 ++++++------ monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index e4a68b6f..eacbb33c 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 9e48f657..15c27a29 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.5 - - idds-workflow==0.7.5 \ No newline at end of file + - idds-common==0.7.6 + - idds-workflow==0.7.6 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index e4a68b6f..eacbb33c 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index f9194802..b2ee4f0d 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.5 - - idds-workflow==0.7.5 \ No newline at end of file + - idds-common==0.7.6 + - idds-workflow==0.7.6 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index e4a68b6f..eacbb33c 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 3b255fe8..1ed87a40 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 24ab1298..9a0d06f6 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.5 - - idds-workflow==0.7.5 \ No newline at end of file + - idds-common==0.7.6 + - idds-workflow==0.7.6 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index e4a68b6f..eacbb33c 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index f565dcba..bf0df2bb 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.5 - - idds-workflow==0.7.5 - - idds-client==0.7.5 \ No newline at end of file + - idds-common==0.7.6 + - idds-workflow==0.7.6 + - idds-client==0.7.6 \ No newline at end of file diff --git a/monitor/conf.js b/monitor/conf.js index e559ac83..410cd4a1 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus739.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus739.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus739.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus740.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus740.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus740.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/version.py b/monitor/version.py index e4a68b6f..eacbb33c 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/website/version.py b/website/version.py index e4a68b6f..eacbb33c 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index e4a68b6f..eacbb33c 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.5" +release_version = "0.7.6" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 5d909319..389b5319 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.5 \ No newline at end of file + - idds-common==0.7.6 \ No newline at end of file From 38b672ff5dd3f53e537c8aa487d5ad23f90dcee5 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 15 Sep 2021 11:10:05 +0200 Subject: [PATCH 075/156] clientmanager submit with username --- client/lib/idds/client/clientmanager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/lib/idds/client/clientmanager.py b/client/lib/idds/client/clientmanager.py index 284e0b8b..f0b58db3 100644 --- a/client/lib/idds/client/clientmanager.py +++ b/client/lib/idds/client/clientmanager.py @@ -38,7 +38,7 @@ def __init__(self, host=None): self.client = Client(host=self.host) @exception_handler - def submit(self, workflow): + def submit(self, workflow, username=None, userdn=None): """ Submit the workflow as a request to iDDS server. @@ -49,8 +49,8 @@ def submit(self, workflow): 'name': workflow.name, 'requester': 'panda', 'request_type': RequestType.Workflow, - 'username': workflow.username, - 'userdn': workflow.userdn, + 'username': username if username else workflow.username, + 'userdn': userdn if userdn else workflow.userdn, 'transform_tag': 'workflow', 'status': RequestStatus.New, 'priority': 0, From 49298bad4c2c495fdf7789a516bb94b3bc5ad5ad Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 15 Sep 2021 11:11:33 +0200 Subject: [PATCH 076/156] new version 0.7.7 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index eacbb33c..a09efc7f 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 15c27a29..072a4144 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.6 - - idds-workflow==0.7.6 \ No newline at end of file + - idds-common==0.7.7 + - idds-workflow==0.7.7 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index eacbb33c..a09efc7f 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index b2ee4f0d..f0bb808f 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.6 - - idds-workflow==0.7.6 \ No newline at end of file + - idds-common==0.7.7 + - idds-workflow==0.7.7 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index eacbb33c..a09efc7f 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 1ed87a40..81cf0408 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 9a0d06f6..1c487dea 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.6 - - idds-workflow==0.7.6 \ No newline at end of file + - idds-common==0.7.7 + - idds-workflow==0.7.7 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index eacbb33c..a09efc7f 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index bf0df2bb..6a121769 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.6 - - idds-workflow==0.7.6 - - idds-client==0.7.6 \ No newline at end of file + - idds-common==0.7.7 + - idds-workflow==0.7.7 + - idds-client==0.7.7 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index eacbb33c..a09efc7f 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/website/version.py b/website/version.py index eacbb33c..a09efc7f 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index eacbb33c..a09efc7f 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.6" +release_version = "0.7.7" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 389b5319..dfbea82f 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.6 \ No newline at end of file + - idds-common==0.7.7 \ No newline at end of file From 9b6e6cefc5034edca168743fb013d08688c3f688 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 25 Sep 2021 14:12:28 +0200 Subject: [PATCH 077/156] fix request query with username --- main/lib/idds/orm/requests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/main/lib/idds/orm/requests.py b/main/lib/idds/orm/requests.py index 669c9920..a9868e4e 100644 --- a/main/lib/idds/orm/requests.py +++ b/main/lib/idds/orm/requests.py @@ -246,6 +246,8 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Request.name, models.Request.requester, models.Request.request_type, + models.Request.username, + models.Request.userdn, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, @@ -288,6 +290,8 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Request.name, models.Request.requester, models.Request.request_type, + models.Request.username, + models.Request.userdn, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, @@ -317,6 +321,8 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Request.name, models.Request.requester, models.Request.request_type, + models.Request.username, + models.Request.userdn, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, @@ -393,6 +399,8 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Request.name, models.Request.requester, models.Request.request_type, + models.Request.username, + models.Request.userdn, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, @@ -430,6 +438,8 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Request.name, models.Request.requester, models.Request.request_type, + models.Request.username, + models.Request.userdn, models.Request.transform_tag, models.Request.workload_id, models.Request.priority, From dc4d04939410e23e771c3f265ba661afa290c50b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 6 Oct 2021 17:15:41 +0200 Subject: [PATCH 078/156] mount cvmfs and add proxy --- .../lib/idds/atlas/workflow/atlascondorwork.py | 4 ++++ atlas/lib/idds/atlas/workflow/atlashpowork.py | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/atlas/lib/idds/atlas/workflow/atlascondorwork.py b/atlas/lib/idds/atlas/workflow/atlascondorwork.py index d6125423..1e2efd69 100644 --- a/atlas/lib/idds/atlas/workflow/atlascondorwork.py +++ b/atlas/lib/idds/atlas/workflow/atlascondorwork.py @@ -101,6 +101,10 @@ def generate_processing_submit_file(self, processing): # self.logger.info("tf_inputs: %s, tf_outputs: %s" % (str(tf_inputs), str(tf_outputs))) + # if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + # proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + # tf_inputs = tf_inputs + [os.environ['X509_USER_PROXY']] + if tf_inputs: jdl += "transfer_input_files = %s\n" % (str(','.join(tf_inputs))) if tf_outputs: diff --git a/atlas/lib/idds/atlas/workflow/atlashpowork.py b/atlas/lib/idds/atlas/workflow/atlashpowork.py index be85c682..11513b23 100644 --- a/atlas/lib/idds/atlas/workflow/atlashpowork.py +++ b/atlas/lib/idds/atlas/workflow/atlashpowork.py @@ -438,6 +438,11 @@ def generate_processing_script_nevergrad(self, processing): 'NUM_POINTS': self.points_to_generate, 'IN': self.input_json, 'OUT': self.output_json} + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + param_values['X509_USER_PROXY_FULLNAME'] = os.environ['X509_USER_PROXY'] + param_values['X509_USER_PROXY_BASENAME'] = proxy_filename + arguments = replace_parameters_with_values(arguments, param_values) script = "#!/bin/bash\n\n" @@ -473,6 +478,12 @@ def generate_processing_script_container(self, processing): 'NUM_POINTS': self.points_to_generate, 'IN': self.input_json, 'OUT': self.output_json} + proxy_filename = 'x509up' + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + param_values['X509_USER_PROXY_FULLNAME'] = os.environ['X509_USER_PROXY'] + param_values['X509_USER_PROXY_BASENAME'] = proxy_filename + executable = replace_parameters_with_values(self.executable, param_values) arguments = replace_parameters_with_values(self.arguments, param_values) @@ -492,7 +503,7 @@ def generate_processing_script_container(self, processing): script += "\n" if self.sandbox and 'docker' in executable: - arguments = 'run --rm -v $(pwd):%s %s ' % (self.container_workdir, self.sandbox) + arguments + arguments = 'run --rm -v $(pwd):%s -v /cvmfs:/cvmfs -e X509_USER_PROXY=%s/%s %s ' % (self.container_workdir, self.container_workdir, proxy_filename, self.sandbox) + arguments script += "echo '%s' '%s'\n" % (str(executable), str(arguments)) script += '%s %s\n' % (str(executable), str(arguments)) @@ -515,6 +526,11 @@ def generate_processing_script_sandbox(self, processing): 'NUM_POINTS': self.points_to_generate, 'IN': self.input_json, 'OUT': self.output_json} + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + param_values['X509_USER_PROXY_FULLNAME'] = os.environ['X509_USER_PROXY'] + param_values['X509_USER_PROXY_BASENAME'] = proxy_filename + executable = replace_parameters_with_values(self.executable, param_values) arguments = replace_parameters_with_values(self.arguments, param_values) From 9cef6d99db1368e8bf12c067a36a99e3dd404895 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 7 Oct 2021 18:11:30 +0200 Subject: [PATCH 079/156] update uuid to short --- workflow/lib/idds/workflow/work.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index d5fa182c..f6829226 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -57,7 +57,7 @@ def __init__(self, scope=None, name=None, coll_type=CollectionType.Dataset, coll self.collection = None - self.internal_id = str(uuid.uuid1()) + self.internal_id = str(uuid.uuid4())[:8] self.coll_id = None self.coll_type = coll_type self.status = CollectionStatus.New @@ -139,7 +139,7 @@ def __init__(self, processing_metadata={}): self.processing = None - self.internal_id = str(uuid.uuid1()) + self.internal_id = str(uuid.uuid4())[:8] self.processing_id = None self.workload_id = None self.status = ProcessingStatus.New @@ -399,7 +399,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, """ super(Work, self).__init__() - self.internal_id = str(uuid.uuid1()) + self.internal_id = str(uuid.uuid4())[:8] self.template_work_id = self.internal_id self.is_template = is_template self.class_name = self.__class__.__name__.lower() @@ -1097,7 +1097,7 @@ def generate_work_from_template(self): new_work.logger = logger # new_work.template_work_id = self.get_internal_id() if self.is_template: - new_work.internal_id = str(uuid.uuid1()) + new_work.internal_id = str(uuid.uuid4())[:8] return new_work def get_template_id(self): From 51e77dbd96e63bdb0723ab40b1c0775e7168cd38 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 00:17:23 +0200 Subject: [PATCH 080/156] fix to avoid change orig attributes during json dumping --- common/lib/idds/common/dict_class.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/lib/idds/common/dict_class.py b/common/lib/idds/common/dict_class.py index c4f8c759..bbceacd0 100644 --- a/common/lib/idds/common/dict_class.py +++ b/common/lib/idds/common/dict_class.py @@ -26,8 +26,10 @@ def to_dict_l(self, d): if hasattr(d, 'to_dict'): return d.to_dict() elif isinstance(d, dict): + new_d = {} for k, v in d.items(): - d[k] = self.to_dict_l(v) + new_d[k] = self.to_dict_l(v) + return new_d elif isinstance(d, list): new_d = [] for k in d: @@ -48,10 +50,10 @@ def to_dict(self): # if not key.startswith('__') and not key.startswith('_'): if not key.startswith('__'): if key == 'logger': - value = None + new_value = None else: - value = self.to_dict_l(value) - ret['attributes'][key] = value + new_value = self.to_dict_l(value) + ret['attributes'][key] = new_value return ret @staticmethod From c5811d0675efb28997fbcc68a49cbfa05fd742d4 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 00:19:06 +0200 Subject: [PATCH 081/156] fix messaging to only record succcessfuly sent messages --- atlas/lib/idds/atlas/notifier/messaging.py | 5 +++++ main/lib/idds/agents/conductor/conductor.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/atlas/lib/idds/atlas/notifier/messaging.py b/atlas/lib/idds/atlas/notifier/messaging.py index b895b73b..9f0442ff 100644 --- a/atlas/lib/idds/atlas/notifier/messaging.py +++ b/atlas/lib/idds/atlas/notifier/messaging.py @@ -56,6 +56,7 @@ def __init__(self, **kwargs): self.setup_logger() self.graceful_stop = threading.Event() self.request_queue = None + self.output_queue = None if not hasattr(self, 'brokers'): raise Exception('brokers is required but not defined.') @@ -80,6 +81,9 @@ def stop(self): def set_request_queue(self, request_queue): self.request_queue = request_queue + def set_output_queue(self, output_queue): + self.output_queue = output_queue + def connect_to_messaging_brokers(self): broker_addresses = [] for b in self.brokers: @@ -126,6 +130,7 @@ def run(self): msg = self.request_queue.get(False) if msg: self.send_message(msg) + self.output_queue.put(msg) else: time.sleep(1) except Exception as error: diff --git a/main/lib/idds/agents/conductor/conductor.py b/main/lib/idds/agents/conductor/conductor.py index 50773af2..f1d58101 100644 --- a/main/lib/idds/agents/conductor/conductor.py +++ b/main/lib/idds/agents/conductor/conductor.py @@ -37,6 +37,7 @@ def __init__(self, num_threads=1, retrieve_bulk_size=None, **kwargs): self.config_section = Sections.Conductor self.retrieve_bulk_size = int(retrieve_bulk_size) self.message_queue = Queue() + self.output_message_queue = Queue() def __del__(self): self.stop_notifier() @@ -70,6 +71,7 @@ def start_notifier(self): self.logger.info("Starting notifier: %s" % self.notifier) self.notifier.set_request_queue(self.message_queue) + self.notifier.set_output_queue(self.output_message_queue) self.notifier.start() def stop_notifier(self): @@ -77,6 +79,17 @@ def stop_notifier(self): self.logger.info("Stopping notifier: %s" % self.notifier) self.notifier.stop() + def get_output_messages(self): + msgs = [] + try: + while not self.output_message_queue.empty(): + msg = self.output_message_queue.get(False) + if msg: + msgs.append(msg) + except Exception as error: + self.logger.error("Failed to get output messages: %s, %s" % (error, traceback.format_exc())) + return msgs + def run(self): """ Main run function. @@ -99,7 +112,8 @@ def run(self): self.message_queue.put(message) while not self.message_queue.empty(): time.sleep(1) - self.clean_messages(messages) + output_messages = self.get_output_messages() + self.clean_messages(output_messages) except IDDSException as error: self.logger.error("Main thread IDDSException: %s" % str(error)) except Exception as error: From 620fb96f705961dd8ddb55703abd754212956390 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 00:20:59 +0200 Subject: [PATCH 082/156] fix run archive --- .../idds/atlas/workflow/atlascondorwork.py | 4 +-- main/lib/idds/agents/archive/run_archive.py | 2 +- main/lib/idds/tests/core_tests.py | 24 +++++++++---- main/lib/idds/tests/panda_test.py | 19 +++++------ main/lib/idds/tests/trigger_release.py | 34 ++++++++++--------- 5 files changed, 48 insertions(+), 35 deletions(-) diff --git a/atlas/lib/idds/atlas/workflow/atlascondorwork.py b/atlas/lib/idds/atlas/workflow/atlascondorwork.py index 1e2efd69..b1d70d81 100644 --- a/atlas/lib/idds/atlas/workflow/atlascondorwork.py +++ b/atlas/lib/idds/atlas/workflow/atlascondorwork.py @@ -102,8 +102,8 @@ def generate_processing_submit_file(self, processing): # self.logger.info("tf_inputs: %s, tf_outputs: %s" % (str(tf_inputs), str(tf_outputs))) # if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: - # proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) - # tf_inputs = tf_inputs + [os.environ['X509_USER_PROXY']] + # proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + # tf_inputs = tf_inputs + [os.environ['X509_USER_PROXY']] if tf_inputs: jdl += "transfer_input_files = %s\n" % (str(','.join(tf_inputs))) diff --git a/main/lib/idds/agents/archive/run_archive.py b/main/lib/idds/agents/archive/run_archive.py index 141e62f0..1713c253 100644 --- a/main/lib/idds/agents/archive/run_archive.py +++ b/main/lib/idds/agents/archive/run_archive.py @@ -27,7 +27,7 @@ def get_archive_sql(schema): updated_at, next_poll_at, accessed_at, expired_at, errors, request_metadata, processing_metadata FROM {schema}.requests - WHERE status in (3, 5, 9, 17) and created_at < sysdate - interval '6' month + WHERE status in (3, 5, 9, 17) and created_at < sysdate - interval '3' month order by request_id asc) LOOP --- archive records diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index cd3fdd48..288cce41 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,6 +102,7 @@ def show_works(req): print(work_ids) +""" reqs = get_requests(request_id=256, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) @@ -111,6 +112,7 @@ def show_works(req): pass sys.exit(0) +""" # reqs = get_requests() # print(len(reqs)) @@ -123,13 +125,17 @@ def show_works(req): pass """ -tfs = get_transforms(transform_id=53550) +""" +tfs = get_transforms(request_id=301) for tf in tfs: # print(tf) # print(tf['transform_metadata']['work'].to_dict()) - # print(json_dumps(tf, sort_keys=True, indent=4)) + print(json_dumps(tf, sort_keys=True, indent=4)) pass +sys.exit(0) + + msgs = retrieve_messages(workload_id=25972557) number_contents = 0 for msg in msgs: @@ -146,12 +152,18 @@ def show_works(req): print(number_contents) sys.exit(0) +""" -prs = get_processings() +prs = get_processings(request_id=301) +i = 0 for pr in prs: - if pr['request_id'] == 91: - # print(json_dumps(pr, sort_keys=True, indent=4)) - pass + # if pr['request_id'] == 91: + print("processing_number: %s" % i) + i += 1 + print(json_dumps(pr, sort_keys=True, indent=4)) + pass + +sys.exit(0) to_release_inputs = [{'request_id': 248, 'coll_id': 3425, diff --git a/main/lib/idds/tests/panda_test.py b/main/lib/idds/tests/panda_test.py index 4b74a0fd..8cfa1fe4 100644 --- a/main/lib/idds/tests/panda_test.py +++ b/main/lib/idds/tests/panda_test.py @@ -3,11 +3,12 @@ import sys import datetime -os.environ['PANDA_URL'] = 'http://ai-idds-01.cern.ch:25080/server/panda' -os.environ['PANDA_URL_SSL'] = 'https://ai-idds-01.cern.ch:25443/server/panda' +os.environ['PANDA_URL'] = 'http://pandaserver-doma.cern.ch:25080/server/panda' +os.environ['PANDA_URL_SSL'] = 'https://pandaserver-doma.cern.ch:25443/server/panda' from pandatools import Client # noqa E402 +""" jobids = [1408118] jobs_list_status = Client.getJobStatus(jobids, verbose=0) print(jobs_list_status) @@ -35,7 +36,6 @@ ret = Client.getTaskStatus(jediTaskID, verbose=False) print(ret) -""" sys.exit(0) jediTaskID = 998 @@ -70,7 +70,7 @@ task_ids = [] for task_id in task_ids: print("Killing %s" % task_id) - Client.killTask(task_id) + # Client.killTask(task_id) """ jobids = [] @@ -103,11 +103,10 @@ # site = newOpts.get('site', None) # excludedSite = newOpts.get('excludedSite', None) # for JEDI -""" -taskID=26034796 -status, out = Client.retryTask(taskID, verbose=True, properErrorCode=True, newParams=newOpts) -print(status) -print(out) -""" +taskIDs = [7056, 7057] +for taskID in taskIDs: + status, out = Client.retryTask(taskID, verbose=True, properErrorCode=True, newParams=newOpts) + print(status) + print(out) sys.exit(0) diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 70e8dc16..f21db22a 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -11,22 +11,24 @@ from idds.orm.contents import get_input_contents # noqa F401 -request_id = 221 -contents = get_contents(request_id=request_id, status=ContentStatus.Available) -ret_contents = {} -for content in contents: - if content['content_relation_type'] == ContentRelationType.Output: # InputDependency - if content['coll_id'] not in ret_contents: - ret_contents[content['coll_id']] = [] - ret_contents[content['coll_id']].append(content) +request_ids = [368, 369, 370, 371, 372, 373, 374, 375, 376] +request_ids = [419] +for request_id in request_ids: + contents = get_contents(request_id=request_id, status=ContentStatus.Available) + ret_contents = {} + for content in contents: + if content['content_relation_type'] == ContentRelationType.Output: # InputDependency + if content['coll_id'] not in ret_contents: + ret_contents[content['coll_id']] = [] + ret_contents[content['coll_id']].append(content) -for ret_content in ret_contents: - print(ret_content) - break + for ret_content in ret_contents: + print(ret_content) + break -updated_contents = core_transforms.release_inputs_by_collection(ret_contents) -for update_content in updated_contents: - print(update_content) - # break + updated_contents = core_transforms.release_inputs_by_collection(ret_contents) + for update_content in updated_contents: + print(update_content) + # break -update_contents(updated_contents) + update_contents(updated_contents) From 167dce5887bed1c942416f0f3caf454822add790 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 00:21:58 +0200 Subject: [PATCH 083/156] change work transforming status when updating status --- workflow/lib/idds/workflow/work.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index f6829226..8051a079 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -186,6 +186,9 @@ def workload_id(self): def workload_id(self, value): self.add_metadata_item('workload_id', value) + def get_workload_id(self): + return self.workload_id + @property def status(self): return self.get_metadata_item('status', ProcessingStatus.New) @@ -486,6 +489,8 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.backup_to_release_inputs = {'0': [], '1': [], '2': []} + self.num_run = None + """ self._running_data_names = [] for name in ['internal_id', 'template_work_id', 'initialized', 'sequence_id', 'parameters', 'work_id', 'transforming', 'workdir', @@ -527,6 +532,17 @@ def template_work_id(self): def template_work_id(self, value): self.add_metadata_item('template_work_id', value) + @property + def workload_id(self): + return self.get_metadata_item('workload_id', None) + + @workload_id.setter + def workload_id(self, value): + self.add_metadata_item('workload_id', value) + + def get_workload_id(self): + return self.workload_id + @property def initialized(self): return self.get_metadata_item('initialized', False) @@ -605,6 +621,13 @@ def status(self): @status.setter def status(self, value): + if not self.transforming: + if value and value in [WorkStatus.Transforming, + WorkStatus.Finished, + WorkStatus.SubFinished, + WorkStatus.Failed, + WorkStatus.Running]: + self.transforming = True self.add_metadata_item('status', value) @property From 3eedf892a5cecdeac50ec50e3912d0d8035e7db3 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 00:23:43 +0200 Subject: [PATCH 084/156] new workflow v2 --- workflow/lib/idds/workflow/workflow1.py | 1655 ++++++++++++++++++ workflow/lib/idds/workflow/workflowv2.py | 1713 +++++++++++++++++++ workflow/lib/idds/workflowv2/__init__.py | 9 + workflow/lib/idds/workflowv2/base.py | 86 + workflow/lib/idds/workflowv2/version.py | 12 + workflow/lib/idds/workflowv2/work.py | 1932 ++++++++++++++++++++++ workflow/lib/idds/workflowv2/workflow.py | 1825 ++++++++++++++++++++ 7 files changed, 7232 insertions(+) create mode 100644 workflow/lib/idds/workflow/workflow1.py create mode 100644 workflow/lib/idds/workflow/workflowv2.py create mode 100644 workflow/lib/idds/workflowv2/__init__.py create mode 100644 workflow/lib/idds/workflowv2/base.py create mode 100644 workflow/lib/idds/workflowv2/version.py create mode 100644 workflow/lib/idds/workflowv2/work.py create mode 100644 workflow/lib/idds/workflowv2/workflow.py diff --git a/workflow/lib/idds/workflow/workflow1.py b/workflow/lib/idds/workflow/workflow1.py new file mode 100644 index 00000000..e0f60f8d --- /dev/null +++ b/workflow/lib/idds/workflow/workflow1.py @@ -0,0 +1,1655 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + +import copy +import datetime +import logging +import inspect +import random +import time +import uuid + + +from idds.common import exceptions +from idds.common.constants import IDDSEnum +from idds.common.utils import json_dumps, setup_logging, get_proxy +from idds.common.utils import str_to_date +from .base import Base +from .work import Work + + +setup_logging(__name__) + + +class ConditionOperator(IDDSEnum): + And = 0 + Or = 1 + + +class ConditionTrigger(IDDSEnum): + NotTriggered = 0 + ToTrigger = 1 + Triggered = 2 + + +class CompositeCondition(Base): + def __init__(self, operator=ConditionOperator.And, conditions=[], true_works=None, false_works=None, logger=None): + self._conditions = [] + self._true_works = [] + self._false_works = [] + + super(CompositeCondition, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_id = self.internal_id + # self.template_id = str(uuid.uuid4())[:8] + + self.logger = logger + if self.logger is None: + self.setup_logger() + + if conditions is None: + conditions = [] + if true_works is None: + true_works = [] + if false_works is None: + false_works = [] + if conditions and type(conditions) not in [tuple, list]: + conditions = [conditions] + if true_works and type(true_works) not in [tuple, list]: + true_works = [true_works] + if false_works and type(false_works) not in [tuple, list]: + false_works = [false_works] + self.validate_conditions(conditions) + + self.operator = operator + self.conditions = [] + self.true_works = [] + self.false_works = [] + + self.conditions = conditions + self.true_works = true_works + self.false_works = false_works + + def get_class_name(self): + return self.__class__.__name__ + + def get_internal_id(self): + return self.internal_id + + def get_template_id(self): + return self.template_id + + @property + def conditions(self): + # return self.get_metadata_item('true_works', []) + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + new_value = [] + for cond in value: + if cond is None: + continue + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id(), + 'idds_method_class_id': cond.__self__.get_template_id()} + else: + new_cond = cond + new_value.append(new_cond) + self.add_metadata_item('conditions', new_value) + + @property + def true_works(self): + # return self.get_metadata_item('true_works', []) + return self._true_works + + @true_works.setter + def true_works(self, value): + self._true_works = value + true_work_meta = self.get_metadata_item('true_works', {}) + for work in value: + if work is None: + continue + if isinstance(work, Work): + if work.get_template_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id()} + elif isinstance(work, CompositeCondition): + if work.get_template_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id(), + 'metadata': work.metadata} + elif isinstance(work, Workflow): + if work.get_template_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id(), + 'metadata': work.metadata} + self.add_metadata_item('true_works', true_work_meta) + + @property + def false_works(self): + # return self.get_metadata_item('false_works', []) + return self._false_works + + @false_works.setter + def false_works(self, value): + self._false_works = value + false_work_meta = self.get_metadata_item('false_works', {}) + for work in value: + if work is None: + continue + if isinstance(work, Work): + if work.get_template_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id()} + elif isinstance(work, CompositeCondition): + if work.get_template_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id(), + 'metadata': work.metadata} + elif isinstance(work, Workflow): + if work.get_template_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id(), + 'metadata': work.metadata} + self.add_metadata_item('false_works', false_work_meta) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + for cond in conditions: + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + if (cond.__self__.is_template): + raise exceptions.IDDSException("Work class for CompositeCondition must not be a template") + + def add_condition(self, cond): + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + if (cond.__self__.is_template): + raise exceptions.IDDSException("Work class for CompositeCondition must not be a template") + + # self.conditions.append({'condition': cond, 'current_work': cond.__self__}) + + self._conditions.append(cond) + new_value = self.get_metadata_item('conditions', []) + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id(), + 'idds_method_class_id': cond.__self__.get_template_id()} + else: + new_cond = cond + new_value.append(new_cond) + self.add_metadata_item('conditions', new_value) + + def load_metadata(self): + # conditions = self.get_metadata_item('conditions', []) + true_works_meta = self.get_metadata_item('true_works', {}) + false_works_meta = self.get_metadata_item('false_works', {}) + + for work in self.true_works: + if isinstance(work, CompositeCondition) or isinstance(work, Workflow): + if work.get_internal_id() in true_works_meta: + work.metadata = true_works_meta[work.get_internal_id()]['metadata'] + for work in self.false_works: + if isinstance(work, CompositeCondition) or isinstance(work, Workflow): + if work.get_internal_id() in false_works_meta: + work.metadata = false_works_meta[work.get_internal_id()]['metadata'] + + def to_dict(self): + # print('to_dict') + ret = {'class': self.__class__.__name__, + 'module': self.__class__.__module__, + 'attributes': {}} + for key, value in self.__dict__.items(): + # print(key) + # print(value) + # if not key.startswith('__') and not key.startswith('_'): + if not key.startswith('__'): + if key == 'logger': + value = None + elif key == '_conditions': + new_value = [] + for cond in value: + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id(), + 'idds_method_class_id': cond.__self__.get_template_id()} + else: + new_cond = cond + new_value.append(new_cond) + value = new_value + elif key in ['_true_works', '_false_works']: + new_value = [] + for w in value: + if isinstance(w, Work): + new_w = w.get_template_id() + elif isinstance(w, CompositeCondition): + new_w = w.to_dict() + elif isinstance(w, Workflow): + new_w = w.to_dict() + else: + new_w = w + new_value.append(new_w) + value = new_value + else: + value = self.to_dict_l(value) + ret['attributes'][key] = value + return ret + + def get_work_from_id(self, work_id, class_id, works, works_template): + for w_id in works: + if works[w_id].get_internal_id() == work_id: + return works[w_id] + for w_id in works_template: + if works_template[w_id].get_template_id() == class_id: + return works_template[w_id] + return None + + def load_conditions(self, works, works_template): + new_conditions = [] + for cond in self.conditions: + if 'idds_method' in cond and 'idds_method_class_id' in cond: + internal_id = cond['idds_method_internal_id'] + class_id = cond['idds_method_class_id'] + work = self.get_work_from_id(internal_id, class_id, works, works_template) + if work is not None: + new_cond = getattr(work, cond['idds_method']) + else: + self.logger.error("Work cannot be found for %s:%s" % (internal_id, class_id)) + new_cond = cond + else: + new_cond = cond + new_conditions.append(new_cond) + self.conditions = new_conditions + + new_true_works = [] + for w in self.true_works: + class_id = self.true_works[w]['template_id'] + if isinstance(w, CompositeCondition): + # work = w.load_conditions(works, works_template) + w.load_conditions(works, works_template) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, class_id, works, works_template) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_true_works.append(work) + self.true_works = new_true_works + + new_false_works = [] + for w in self.false_works: + class_id = self.true_works[w]['template_id'] + if isinstance(w, CompositeCondition): + # work = w.load_condtions(works, works_template) + w.load_conditions(works, works_template) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, class_id, works, works_template) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_false_works.append(work) + self.false_works = new_false_works + + def all_works(self): + works = [] + works = works + self.all_pre_works() + works = works + self.all_next_works() + return works + + def all_condition_ids(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__.get_internal_id()) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_condition_ids() + return works + + def all_pre_works(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_pre_works() + return works + + def all_next_works(self): + works = [] + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_next_works() + else: + works.append(work) + return works + + def get_current_cond_status(self, cond): + if callable(cond): + if cond(): + return True + else: + return False + else: + if cond: + return True + else: + return False + + def get_cond_status(self): + if self.operator == ConditionOperator.And: + for cond in self.conditions: + if not self.get_current_cond_status(cond): + return False + return True + else: + for cond in self.conditions: + if self.get_current_cond_status(cond): + return True + return False + + def get_condition_status(self): + return self.get_cond_status() + + def is_condition_true(self): + if self.get_cond_status(): + return True + return False + + def is_condition_false(self): + if not self.get_cond_status(): + return True + return False + + def get_next_works(self, trigger=ConditionTrigger.NotTriggered): + works = [] + if self.get_cond_status(): + true_work_meta = self.get_metadata_item('true_works', {}) + for work in self.true_works: + if isinstance(work, CompositeCondition) or isinstance(work, Workflow): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id()} + if trigger == ConditionTrigger.ToTrigger: + if not true_work_meta[work.get_internal_id()]['triggered']: + true_work_meta[work.get_internal_id()]['triggered'] = True + works.append(work) + elif work.get_is_template(): + # A template can be triggered many times. + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not true_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if true_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + self.add_metadata_item('true_works', true_work_meta) + else: + false_work_meta = self.get_metadata_item('false_works', {}) + for work in self.false_works: + if isinstance(work, CompositeCondition) or isinstance(work, Workflow): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False, + 'template_id': work.get_template_id()} + if trigger == ConditionTrigger.ToTrigger: + if not false_work_meta[work.get_internal_id()]['triggered']: + false_work_meta[work.get_internal_id()]['triggered'] = True + works.append(work) + elif work.get_is_template(): + # A template can be triggered many times. + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not false_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if false_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + self.add_metadata_item('false_works', false_work_meta) + return works + + def generate_condition_from_template(self): + logger = self.logger + self.logger = None + new_cond = copy.deepcopy(self) + self.logger = logger + new_cond.logger = logger + # new_work.template_work_id = self.get_internal_id() + new_cond.internal_id = str(uuid.uuid4())[:8] + return new_cond + + +class AndCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(AndCondition, self).__init__(operator=ConditionOperator.And, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class OrCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(OrCondition, self).__init__(operator=ConditionOperator.Or, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class Condition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + super(Condition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + # to support load from old conditions + @property + def cond(self): + # return self.get_metadata_item('true_works', []) + return self.conditions[0] if len(self.conditions) >= 1 else None + + @cond.setter + def cond(self, value): + self.conditions = [value] + + @property + def true_work(self): + # return self.get_metadata_item('true_works', []) + return self.true_works if len(self.true_works) >= 1 else None + + @true_work.setter + def true_work(self, value): + self.true_works = [value] + + @property + def false_work(self): + # return self.get_metadata_item('true_works', []) + return self.false_works if len(self.false_works) >= 1 else None + + @false_work.setter + def false_work(self, value): + self.false_works = [value] + + +class TemplateCondition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + if true_work is not None and not isinstance(true_work, Work): + raise exceptions.IDDSException("true_work can only be set with Work class") + if false_work is not None and not isinstance(false_work, Work): + raise exceptions.IDDSException("false_work can only be set with Work class") + + super(TemplateCondition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + if len(conditions) > 1: + raise exceptions.IDDSException("Condition class can only support one condition. To support multiple condition, please use CompositeCondition.") + for cond in conditions: + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + + def add_condition(self, cond): + raise exceptions.IDDSException("Condition class doesn't support add_condition. To support multiple condition, please use CompositeCondition.") + + +class ParameterLink(Base): + def __init__(self, parameters): + self.parameters = parameters + + def get_parameter_value(self, work, p): + p_f = getattr(work, p, 'None') + if p_f: + if callable(p_f): + return p_f() + else: + return p_f + else: + return None + + def set_parameters(self, work): + p_values = {} + for p in self.parameters: + p_values[p] = self.get_parameter_value(work, p) + self.add_metadata_item('parameters', p_values) + + def get_parameters(self): + return self.get_metadata_item('parameters', {}) + + +class Workflow(Base): + + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + """ + Init a workflow. + """ + self._conditions = {} + self._conditions_temp = {} + self._work_conds = {} + self.conditions_template = {} + self.work_conds_template = {} + + self._parameter_links = {} + + super(Workflow, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_work_id = self.internal_id + # self.template_work_id = str(uuid.uuid4())[:8] + self.lifetime = lifetime + self.pending_time = pending_time + + if name: + self._name = name + "." + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + else: + self._name = 'idds.workflow.' + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + + if workload_id is None: + workload_id = int(time.time()) + self.workload_id = workload_id + + self.logger = logger + if self.logger is None: + self.setup_logger() + + self.works_template = {} + self._works = {} + self.works = {} + self.work_sequence = {} # order list + + self.terminated_works = [] + self.initial_works = [] + # if the primary initial_work is not set, it's the first initial work. + self.primary_initial_work = None + self.independent_works = [] + + self.first_initial = False + self.new_to_run_works = [] + self.current_running_works = [] + + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + self.num_total_works = 0 + + self.last_work = None + + self.last_updated_at = datetime.datetime.utcnow() + self.expired = False + + self.to_update_transforms = {} + + # user defined Condition class + self.user_defined_conditions = {} + + self.username = None + self.userdn = None + self.proxy = None + + self.loop_condition_template = {} + self.loop_condition = None + + self.parameter_links_template = {} + self.parameter_links = {} + + """ + self._running_data_names = [] + for name in ['internal_id', 'template_work_id', 'workload_id', 'work_sequence', 'terminated_works', + 'first_initial', 'new_to_run_works', 'current_running_works', + 'num_subfinished_works', 'num_finished_works', 'num_failed_works', 'num_cancelled_works', 'num_suspended_works', + 'num_expired_works', 'num_total_works', 'last_work']: + self._running_data_names.append(name) + for name in ['works']: + self._running_data_names.append(name) + """ + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + + @property + def internal_id(self): + return self.get_metadata_item('internal_id') + + @internal_id.setter + def internal_id(self, value): + self.add_metadata_item('internal_id', value) + + @property + def template_work_id(self): + return self.get_metadata_item('template_work_id') + + @template_work_id.setter + def template_work_id(self, value): + self.add_metadata_item('template_work_id', value) + + def get_template_work_id(self): + return self.template_work_id + + def get_template_id(self): + return self.template_work_id + + @property + def workload_id(self): + return self.get_metadata_item('workload_id') + + @workload_id.setter + def workload_id(self, value): + self.add_metadata_item('workload_id', value) + + @property + def lifetime(self): + # return self.get_metadata_item('lifetime', None) + return getattr(self, '_lifetime', None) + + @lifetime.setter + def lifetime(self, value): + # self.add_metadata_item('lifetime', value) + self._lifetime = value + + @property + def pending_time(self): + # return self.get_metadata_item('pending_time', None) + return getattr(self, '_pending_time', None) + + @pending_time.setter + def pending_time(self, value): + # self.add_metadata_item('pending_time', value) + self._pending_time = value + + @property + def last_updated_at(self): + last_updated_at = self.get_metadata_item('last_updated_at', None) + if last_updated_at and type(last_updated_at) in [str]: + last_updated_at = str_to_date(last_updated_at) + return last_updated_at + + @last_updated_at.setter + def last_updated_at(self, value): + self.add_metadata_item('last_updated_at', value) + + def has_new_updates(self): + self.last_updated_at = datetime.datetime.utcnow() + + @property + def expired(self): + t = self.get_metadata_item('expired', False) + if type(t) in [bool]: + return t + elif type(t) in [str] and t.lower() in ['true']: + return True + else: + return False + + @expired.setter + def expired(self, value): + self.add_metadata_item('expired', value) + + @property + def works(self): + return self._works + + @works.setter + def works(self, value): + self._works = value + work_metadata = {} + if self._works: + for k in self._works: + work = self._works[k] + if isinstance(work, Workflow): + work_metadata[k] = {'internal_id': work.internal_id, + 'template_id': work.get_template_id(), + 'type': 'workflow', + 'metadata': work.metadata} + else: + work_metadata[k] = {'internal_id': work.internal_id, + 'template_id': work.get_template_id(), + 'type': 'work', + 'work_id': work.work_id, + 'status': work.status, + 'substatus': work.substatus, + 'transforming': work.transforming} + self.add_metadata_item('works', work_metadata) + + def refresh_works(self): + work_metadata = {} + if self._works: + for k in self._works: + work = self._works[k] + if isinstance(work, Workflow): + work.refresh_works() + work_metadata[k] = {'internal_id': work.internal_id, + 'template_id': work.get_template_id(), + 'type': 'workflow', + 'metadata': work.metadata} + else: + work_metadata[k] = {'internal_id': work.internal_id, + 'template_id': work.get_template_id(), + 'type': 'work', + 'work_id': work.work_id, + 'status': work.status, + 'substatus': work.substatus, + 'transforming': work.transforming} + if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): + self.last_updated_at = work.last_updated_at + self.add_metadata_item('works', work_metadata) + + def load_works(self): + work_metadata = self.get_metadata_item('works', {}) + for k in work_metadata: + if 'type' not in work_metadata[k]: + work_metadata[k]['type'] = 'work' + + if work_metadata[k]['type'] == 'work': + if k not in self._works: + template_id = work_metadata[k]['template_id'] + work_template = self.works_template[template_id] + new_work = work_template.generate_work_from_template() + new_work.work_id = work_metadata[k]['work_id'] + new_work.internal_id = work_metadata[k]['internal_id'] + self._works[k] = new_work + self._works[k].work_id = work_metadata[k]['work_id'] + self._works[k].transforming = work_metadata[k]['transforming'] + if 'status' in work_metadata[k]: + self._works[k].status = work_metadata[k]['status'] + self._works[k].substatus = work_metadata[k]['substatus'] + elif work_metadata[k]['type'] == 'workflow': + if k not in self._works: + template_id = work_metadata[k]['template_id'] + workflow_template = self.works_template[template_id] + new_workflow = workflow_template.generate_work_from_template() + new_workflow.metadata = work_metadata[k]['metadata'] + # new_workflow.load_works() + new_workflow.internal_id = work_metadata[k]['internal_id'] + self._works[k] = new_workflow + + work = self._works[k] + if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): + self.last_updated_at = work.last_updated_at + + @property + def conditions(self): + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + conditions_metadata = {} + if self._conditions: + for k in self._conditions: + conditions_metadata[k] = {'template_id': self._conditions[k].get_template_id(), + 'metadata': self._conditions[k].metadata} + self.add_metadata_item('conditions', conditions_metadata) + + @property + def conditions_temp(self): + return self._conditions_temp + + @conditions_temp.setter + def conditions_temp(self, value): + self._conditions_temp = value + conditions_metadata = {} + if self._conditions_temp: + conditions_metadata = self._conditions_temp + self.add_metadata_item('conditions_temp', conditions_metadata) + + @property + def work_conds(self): + return self._work_conds + + @work_conds.setter + def work_conds(self, value): + self._work_conds = value + self.add_metadata_item('work_conds', value) + + def load_work_conditions(self): + conditions_metadata = self.get_metadata_item('conditions', {}) + for cond_internal_id in conditions_metadata: + template_id = conditions_metadata[cond_internal_id]['template_id'] + cond_template = self.conditions_template[template_id] + cond = cond_template.generate_condition_from_template() + cond.metadata = conditions_metadata[cond_internal_id]['metadata'] + self.conditions[cond_internal_id] = cond + self.conditions[cond_internal_id].load_conditions(self.works, self.get_works_template()) + + work_conds = self.get_metadata_item('work_conds', {}) + self._work_conds = work_conds + + @property + def loop_condition(self): + return self._loop_condition + + @loop_condition.setter + def loop_condition(self, value): + self._loop_condition = value + self.add_metadata_item('loop_condition', self._loop_condition.get_condition_status()) + + @property + def work_sequence(self): + return self.get_metadata_item('work_sequence', {}) + + @work_sequence.setter + def work_sequence(self, value): + self.add_metadata_item('work_sequence', value) + + @property + def terminated_works(self): + return self.get_metadata_item('terminated_works', []) + + @terminated_works.setter + def terminated_works(self, value): + self.add_metadata_item('terminated_works', value) + + @property + def first_initial(self): + return self.get_metadata_item('first_initial', False) + + @first_initial.setter + def first_initial(self, value): + self.add_metadata_item('first_initial', value) + + @property + def new_to_run_works(self): + return self.get_metadata_item('new_to_run_works', []) + + @new_to_run_works.setter + def new_to_run_works(self, value): + self.add_metadata_item('new_to_run_works', value) + + @property + def current_running_works(self): + return self.get_metadata_item('current_running_works', []) + + @current_running_works.setter + def current_running_works(self, value): + self.add_metadata_item('current_running_works', value) + + @property + def num_subfinished_works(self): + return self.get_metadata_item('num_subfinished_works', 0) + + @num_subfinished_works.setter + def num_subfinished_works(self, value): + self.add_metadata_item('num_subfinished_works', value) + + @property + def num_finished_works(self): + return self.get_metadata_item('num_finished_works', 0) + + @num_finished_works.setter + def num_finished_works(self, value): + self.add_metadata_item('num_finished_works', value) + + @property + def num_failed_works(self): + return self.get_metadata_item('num_failed_works', 0) + + @num_failed_works.setter + def num_failed_works(self, value): + self.add_metadata_item('num_failed_works', value) + + @property + def num_cancelled_works(self): + return self.get_metadata_item('num_cancelled_works', 0) + + @num_cancelled_works.setter + def num_cancelled_works(self, value): + self.add_metadata_item('num_cancelled_works', value) + + @property + def num_suspended_works(self): + return self.get_metadata_item('num_suspended_works', 0) + + @num_suspended_works.setter + def num_suspended_works(self, value): + self.add_metadata_item('num_suspended_works', value) + + @property + def num_expired_works(self): + return self.get_metadata_item('num_expired_works', 0) + + @num_expired_works.setter + def num_expired_works(self, value): + self.add_metadata_item('num_expired_works', value) + + @property + def num_total_works(self): + return self.get_metadata_item('num_total_works', 0) + + @num_total_works.setter + def num_total_works(self, value): + self.add_metadata_item('num_total_works', value) + + @property + def last_work(self): + return self.get_metadata_item('last_work', None) + + @last_work.setter + def last_work(self, value): + self.add_metadata_item('last_work', value) + + @property + def to_update_transforms(self): + return self.get_metadata_item('to_update_transforms', {}) + + @to_update_transforms.setter + def to_update_transforms(self, value): + self.add_metadata_item('to_update_transforms', value) + + def load_metadata(self): + self.load_works() + self.load_work_conditions() + + def get_class_name(self): + return self.__class__.__name__ + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) + + def log_info(self, info): + if self.logger is None: + self.setup_logger() + self.logger.info(info) + + def log_debug(self, info): + if self.logger is None: + self.setup_logger() + self.logger.debug(info) + + def get_internal_id(self): + return self.internal_id + + def copy(self): + new_wf = copy.deepcopy(self) + return new_wf + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + def get_works_template(self): + return self.works_template + + def generate_work_from_template(self): + logger = self.logger + self.logger = None + new_workflow = copy.deepcopy(self) + self.logger = logger + new_workflow.logger = logger + # new_work.template_work_id = self.get_internal_id() + new_workflow.internal_id = str(uuid.uuid4())[:8] + return new_workflow + + def add_work_template(self, work): + self.works_template[work.get_template_id()] = work + + def get_new_work_from_template(self, work_id, new_parameters=None): + # 1. initialize works + # template_id = work.get_template_id() + template_id = work_id + work = self.works_template[template_id] + new_work = work.generate_work_from_template() + if new_parameters: + new_work.set_parameters(new_parameters) + new_work.sequence_id = self.num_total_works + if isinstance(new_work, Workflow): + pass + else: + new_work.initialize_work() + works = self.works + works[new_work.get_internal_id()] = new_work + self.works = works + # self.work_sequence.append(new_work.get_internal_id()) + self.work_sequence[str(self.num_total_works)] = new_work.get_internal_id() + if isinstance(new_work, Workflow): + self.num_total_works += 1 + else: + self.num_total_works += 1 + self.new_to_run_works.append(new_work.get_internal_id()) + self.last_work = new_work.get_internal_id() + + # 2. initialize conditions related to this work + if template_id in self.work_conds_template: + conds = self.work_conds_template[template_id] + self.work_conds[new_work.get_internal_id()] = [] + for cond_template_id in conds: + cond_template = self.conditions_template[cond_template_id] + if not cond_template.has_multiple_pre_works(): + cond = cond_template.generate_new_cond_from_template() + self.conditions[cond.get_internal_id()] = cond + self.work_conds[new_work.get_internal_id()].append(cond.get_internal_id()) + cond.attach_pre_work(new_work) + else: + if cond_template_id in self.conditions_temp: + # condition is already created, for example, for AndCondition or OrCondition + # cond_temp will be cleaned when a new loop is created in LoopWorkflow + cond_internal_id = self.conditions_temp[cond_template_id] + self.work_conds[new_work.get_internal_id()].append(cond_internal_id) + cond = self.conditions[cond_internal_id] + cond.attach_pre_work(new_work) + else: + cond = cond_template.generate_new_cond_from_template() + self.conditions[cond.get_internal_id()] = cond + self.work_conds[new_work.get_internal_id()].append(cond.get_internal_id()) + self.conditions_temp[cond_template_id] = cond.get_internal_id() + cond.attach_pre_work(new_work) + return new_work + + def register_user_defined_condition(self, condition): + cond_src = inspect.getsource(condition) + self.user_defined_conditions[condition.__name__] = cond_src + + def load_user_defined_condition(self): + # try: + # Condition() + # except NameError: + # global Condition + # import Condition + + for cond_src_name in self.user_defined_conditions: + # global cond_src_name + exec(self.user_defined_conditions[cond_src_name]) + + def set_workload_id(self, workload_id): + self.workload_id = workload_id + + def get_workload_id(self): + return self.workload_id + + def add_work(self, work, initial=False, primary=False): + self.first_initial = False + self.add_work_template(work) + if initial: + if primary: + self.primary_initial_work = work.get_template_id() + self.add_initial_works(work) + + self.independent_works.append(work.get_template_id()) + + def add_condition(self, cond): + self.first_initial = False + cond_works = cond.all_works() + for cond_work in cond_works: + assert(cond_work.get_template_id() in self.get_works_template()) + + if cond.get_template_id() not in self.conditions_template: + conditions = self.conditions_template + conditions[cond.get_template_id()] = cond + self.conditions_template = conditions + + # if cond.current_work not in self.work_conds: + # self.work_conds[cond.current_work] = [] + # self.work_conds[cond.current_work].append(cond) + work_conds = self.work_conds_template + for work in cond.all_pre_works(): + if work.get_template_id() not in work_conds: + work_conds[work.get_template_id()] = [] + work_conds[work.get_template_id()].append(cond.get_template_id()) + self.work_conds_template = work_conds + + # if a work is a true_work or false_work of a condition, + # should remove it from independent_works + cond_next_works = cond.all_next_works() + for next_work in cond_next_works: + if next_work.get_template_id() in self.independent_works: + self.independent_works.remove(next_work.get_template_id()) + + def add_initial_works(self, work): + assert(work.get_template_id() in self.get_works_template()) + self.initial_works.append(work.get_template_id()) + if self.primary_initial_work is None: + self.primary_initial_work = work.get_template_id() + + def enable_next_works(self, work, cond): + self.log_debug("Checking Work %s condition: %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + # load_conditions should cover it. + # if cond and self.is_class_method(cond.cond): + # # cond_work_id = self.works[cond.cond['idds_method_class_id']] + # cond.cond = getattr(work, cond.cond['idds_method']) + + self.log_info("Work %s condition: %s" % (work.get_internal_id(), cond.conditions)) + next_works = cond.get_next_works(trigger=ConditionTrigger.ToTrigger) + self.log_info("Work %s condition status %s" % (work.get_internal_id(), cond.get_cond_status())) + self.log_info("Work %s next works %s" % (work.get_internal_id(), str(next_works))) + new_next_works = [] + if next_works is not None: + for next_work in next_works: + new_parameters = work.get_parameters_for_next_task() + new_next_work = self.get_new_work_from_template(next_work.get_template_id(), new_parameters) + work.add_next_work(new_next_work.get_internal_id()) + # cond.add_condition_work(new_next_work) ####### TODO: + new_next_works.append(new_next_work) + return new_next_works + + def add_loop_condition(self, condition, position='end'): + self.loop_condition_template = {'position': position, + 'condition': condition} + + def has_loop_condition(self): + if self.loop_condition_template and 'condition' in self.loop_condition_template: + return True + return False + + def get_loop_condition_status(self): + if self.has_loop_condition(): + cond_template = self.loop_condition_template['condition'] + loop_condition = cond_template.generate_condition_from_template() + loop_condition.load_conditions(self.works, self.get_works_template()) + self.loop_condition = loop_condition + return self.loop_condition.get_condition_status() + return False + + def __str__(self): + return str(json_dumps(self)) + + def get_new_works(self): + """ + *** Function called by Marshaller agent. + + new works to be ready to start + """ + self.sync_works() + return [self.works[k] for k in self.new_to_run_works] + + def get_current_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + return [self.works[k] for k in self.current_running_works] + + def get_all_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + return [self.works[k] for k in self.works] + + def get_primary_initial_collection(self): + """ + *** Function called by Clerk agent. + """ + + if self.primary_initial_work: + return self.get_works_template()[self.primary_initial_work].get_primary_input_collection() + elif self.initial_works: + return self.get_works_template()[self.initial_works[0]].get_primary_input_collection() + elif self.independent_works: + return self.get_works_template()[self.independent_works[0]].get_primary_input_collection() + else: + keys = self.get_works_template().keys() + return self.get_works_template()[keys[0]].get_primary_input_collection() + return None + + def get_dependency_works(self, work_id, depth, max_depth): + if depth > max_depth: + return [] + + deps = [] + for dep_work_id in self.work_dependencies[work_id]: + deps.append(dep_work_id) + l_deps = self.get_dependency_works(dep_work_id, depth + 1, max_depth) + deps += l_deps + deps = list(dict.fromkeys(deps)) + return deps + + def order_independent_works(self): + ind_work_ids = self.independent_works + self.independent_works = [] + self.work_dependencies = {} + for ind_work_id in ind_work_ids: + work = self.works_template[ind_work_id] + self.work_dependencies[ind_work_id] = [] + for ind_work_id1 in ind_work_ids: + if ind_work_id == ind_work_id1: + continue + work1 = self.works_template[ind_work_id1] + if work.depend_on(work1): + self.work_dependencies[ind_work_id].append(ind_work_id1) + self.log_debug('work dependencies 1: %s' % str(self.work_dependencies)) + + max_depth = len(ind_work_ids) + 1 + work_dependencies = copy.deepcopy(self.work_dependencies) + for work_id in work_dependencies: + deps = self.get_dependency_works(work_id, 0, max_depth) + self.work_dependencies[work_id] = deps + self.log_debug('work dependencies 2: %s' % str(self.work_dependencies)) + + while True: + for work_id in self.work_dependencies: + if work_id not in self.independent_works and len(self.work_dependencies[work_id]) == 0: + self.independent_works.append(work_id) + for work_id in self.independent_works: + if work_id in self.work_dependencies: + del self.work_dependencies[work_id] + for work_id in self.work_dependencies: + for in_work_id in self.independent_works: + if in_work_id in self.work_dependencies[work_id]: + self.work_dependencies[work_id].remove(in_work_id) + if not self.work_dependencies: + break + self.log_debug('independent_works: %s' % str(self.independent_works)) + + def first_initialize(self): + # set new_to_run works + if not self.first_initial: + self.first_initial = True + self.order_independent_works() + if self.initial_works: + tostart_works = self.initial_works + elif self.independent_works: + tostart_works = self.independent_works + else: + tostart_works = list(self.get_works_template().keys()) + tostart_works = [tostart_works[0]] + + for work_id in tostart_works: + self.get_new_work_from_template(work_id) + + def sync_works(self): + self.first_initialize() + + self.refresh_works() + + for k in self.works: + work = self.works[k] + self.log_debug("work %s is_terminated(%s:%s)" % (work.get_internal_id(), work.is_terminated(), work.get_status())) + + for work in [self.works[k] for k in self.new_to_run_works]: + if work.transforming: + self.new_to_run_works.remove(work.get_internal_id()) + self.current_running_works.append(work.get_internal_id()) + + for work in [self.works[k] for k in self.current_running_works]: + if isinstance(work, Workflow): + work.sync_works() + if work.get_internal_id() in self.work_conds: + self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + json_dumps(self.work_conds[work.get_internal_id()], sort_keys=True, indent=4))) + for cond_id in self.work_conds[work.get_internal_id()]: + cond = self.conditions[cond_id] + self.log_debug("Work %s has condition dependencie %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + self.enable_next_works(work, cond) + + if work.is_terminated(): + self.log_info("Work %s is terminated(%s)" % (work.get_internal_id(), work.get_status())) + self.log_debug("Work conditions: %s" % json_dumps(self.work_conds, sort_keys=True, indent=4)) + if work.get_template_id() not in self.work_conds: + # has no next work + self.log_info("Work %s has no condition dependencies" % work.get_internal_id()) + self.terminated_works.append(work.get_internal_id()) + self.current_running_works.remove(work.get_internal_id()) + else: + # self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + # json_dumps(self.work_conds[work.get_template_id()], sort_keys=True, indent=4))) + # for cond in self.work_conds[work.get_template_id()]: + # self.enable_next_works(work, cond) + self.terminated_works.append(work.get_internal_id()) + self.current_running_works.remove(work.get_internal_id()) + + if work.is_finished(): + self.num_finished_works += 1 + elif work.is_subfinished(): + self.num_subfinished_works += 1 + elif work.is_failed(): + self.num_failed_works += 1 + elif work.is_expired(): + self.num_expired_works += 1 + elif work.is_cancelled(): + self.num_cancelled_works += 1 + elif work.is_suspended(): + self.num_suspended_works += 1 + log_str = "num_total_works: %s" % self.num_total_works + log_str += ", num_finished_works: %s" % self.num_finished_works + log_str += ", num_subfinished_works: %s" % self.num_subfinished_works + log_str += ", num_failed_works: %s" % self.num_failed_works + log_str += ", num_expired_works: %s" % self.num_expired_works + log_str += ", num_cancelled_works: %s" % self.num_cancelled_works + log_str += ", num_suspended_works: %s" % self.num_suspended_works + self.log_debug(log_str) + + def resume_works(self): + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + + self.last_updated_at = datetime.datetime.utcnow() + + t_works = self.terminated_works + self.terminated_works = [] + self.current_running_works = self.current_running_works + t_works + for work in [self.works[k] for k in self.current_running_works]: + if isinstance(work, Workflow): + work.resume_works() + else: + work.resume_work() + + def clean_works(self): + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + self.num_total_works = 0 + + self.last_updated_at = datetime.datetime.utcnow() + + self.terminated_works = [] + self.current_running_works = [] + self.works = {} + self.work_sequence = {} # order list + + self.first_initial = False + self.new_to_run_works = [] + + def get_exact_workflows(self): + """ + *** Function called by Clerk agent. + + TODO: The primary dataset for the initial work is a dataset with '*'. + workflow.primary_initial_collection = 'some datasets with *' + collections = get_collection(workflow.primary_initial_collection) + wfs = [] + for coll in collections: + wf = self.copy() + wf.name = self.name + "_" + number + wf.primary_initial_collection = coll + wfs.append(wf) + return wfs + """ + return [self] + + def is_terminated(self): + """ + *** Function called by Marshaller agent. + """ + self.sync_works() + if len(self.new_to_run_works) == 0 and len(self.current_running_works) == 0: + return True + return False + + def is_finished(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and self.num_finished_works == self.num_total_works + + def is_subfinished(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_finished_works + self.num_subfinished_works > 0 and self.num_finished_works + self.num_subfinished_works <= self.num_total_works) + + def is_failed(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_failed_works > 0) and (self.num_cancelled_works == 0) and (self.num_suspended_works == 0) and (self.num_expired_works == 0) + + def is_to_expire(self, expired_at=None, pending_time=None, request_id=None): + if self.expired: + # it's already expired. avoid sending duplicated messages again and again. + return False + if expired_at: + if type(expired_at) in [str]: + expired_at = str_to_date(expired_at) + if expired_at < datetime.datetime.utcnow(): + self.logger.info("Request(%s) expired_at(%s) is smaller than utc now(%s), expiring" % (request_id, + expired_at, + datetime.datetime.utcnow())) + return True + + act_pending_time = None + if self.pending_time: + # in days + act_pending_time = float(self.pending_time) + else: + if pending_time: + act_pending_time = float(pending_time) + if act_pending_time: + act_pending_seconds = int(86400 * act_pending_time) + if self.last_updated_at + datetime.timedelta(seconds=act_pending_seconds) < datetime.datetime.utcnow(): + log_str = "Request(%s) last updated at(%s) + pending seconds(%s)" % (request_id, + self.last_updated_at, + act_pending_seconds) + log_str += " is smaller than utc now(%s), expiring" % (datetime.datetime.utcnow()) + self.logger.info(log_str) + return True + + return False + + def is_expired(self): + """ + *** Function called by Marshaller agent. + """ + # return self.is_terminated() and (self.num_expired_works > 0) + return self.is_terminated() and self.expired + + def is_cancelled(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_cancelled_works > 0) + + def is_suspended(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_suspended_works > 0) + + def get_terminated_msg(self): + """ + *** Function called by Marshaller agent. + """ + if self.last_work: + return self.works[self.last_work].get_terminated_msg() + return None + + def add_proxy(self): + self.proxy = get_proxy() + if not self.proxy: + raise Exception("Cannot get local proxy") + + def get_proxy(self): + return self.proxy + + +class LoopWorkflow(Base): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # super(Workflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + self.logger = logger + if self.logger is None: + self.setup_logger() + + self.template = Workflow(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + self.num_run = 0 + self.runs = {} + self.loop_condition_position = 'end' + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) + + @property + def metadata(self): + run_metadata = {'num_run': self.num_run, + 'runs': {}} + for run_id in self.runs: + run_metadata['runs'][run_id] = self.runs[run_id].metadata + return run_metadata + + @metadata.setter + def metadata(self, value): + run_metadata = value + self.num_run = run_metadata['num_run'] + runs = run_metadata['runs'] + for run_id in runs: + self.runs[run_id] = self.template.copy() + self.runs[run_id].metadata = runs[run_id] + + def set_workload_id(self, workload_id): + self.template.workload_id = workload_id + # self.dynamic.workload_id = workload_id + + def get_workload_id(self): + return self.template.workload_id + + def add_work(self, work, initial=False, primary=False): + self.template.add_work(work, initial, primary) + + def add_condition(self, cond): + self.template.add_condition(cond) + + def get_new_works(self): + self.sync_works() + self.runs[str(self.num_run)].get_new_works() + + def get_current_works(self): + self.sync_works() + self.runs[str(self.num_run)].get_current_works() + + def get_all_works(self): + self.sync_works() + self.runs[str(self.num_run)].get_all_works() + + def get_primary_initial_collection(self): + self.runs[str(self.num_run)].get_primary_initial_collection() + + def resume_works(self): + self.runs[str(self.num_run)].resume_works() + + def clean_works(self): + self.runs[str(self.num_run)].clean_works() + + def is_terminated(self): + self.runs[str(self.num_run)].is_terminated() + + def is_finished(self): + self.runs[str(self.num_run)].is_finished() + + def is_failed(self): + self.runs[str(self.num_run)].is_failed() + + def is_expired(self): + self.runs[str(self.num_run)].is_expired() + + def is_cancelled(self): + self.runs[str(self.num_run)].is_cancelled() + + def is_suspended(self): + self.runs[str(self.num_run)].is_suspended() + + def get_terminated_msg(self): + self.runs[str(self.num_run)].get_terminated_msg() + + def add_proxy(self): + self.template.add_proxy() + + def get_proxy(self): + self.template.get_proxy() + + def add_loop_condition(self, condition, position='end'): + if not position or position != 'begin': + position = 'end' + position = 'end' # force position to end currently. position = 'begin' is not supported now. + self.template.add_loop_condition(condition, position=position) + self.loop_condition_position = position + + def sync_works(self): + # position is end. + if str(self.num_run) not in self.runs: + self.runs[str(self.num_run)] = self.template.copy() + + self.runs[str(self.num_run)].sync_works() + if self.runs[str(self.num_run)].is_terminated(): + if self.runs[str(self.num_run)].has_loop_condition(): + if self.runs[str(self.num_run)].get_loop_condition_status(): + self.num_run += 1 + self.runs[str(self.num_run)] = self.template.copy() + + +class SubWorkflow(LoopWorkflow): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + """ + Init a workflow. + """ + super(SubWorkflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + + +class CompositeWorkflow(LoopWorkflow): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + """ + Init a workflow. + """ + super(CompositeWorkflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) diff --git a/workflow/lib/idds/workflow/workflowv2.py b/workflow/lib/idds/workflow/workflowv2.py new file mode 100644 index 00000000..e2a75615 --- /dev/null +++ b/workflow/lib/idds/workflow/workflowv2.py @@ -0,0 +1,1713 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + +import copy +import datetime +import logging +import inspect +import random +import time +import uuid + + +from idds.common import exceptions +from idds.common.constants import IDDSEnum, WorkStatus +from idds.common.utils import json_dumps, setup_logging, get_proxy +from idds.common.utils import str_to_date +from .base import Base +from .work import Work + + +setup_logging(__name__) + + +class ConditionOperator(IDDSEnum): + And = 0 + Or = 1 + + +class ConditionTrigger(IDDSEnum): + NotTriggered = 0 + ToTrigger = 1 + Triggered = 2 + + +class CompositeCondition(Base): + def __init__(self, operator=ConditionOperator.And, conditions=[], true_works=None, false_works=None, logger=None): + self._conditions = [] + self._true_works = [] + self._false_works = [] + + super(CompositeCondition, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_id = self.internal_id + # self.template_id = str(uuid.uuid4())[:8] + + self.logger = logger + if self.logger is None: + self.setup_logger() + + if conditions is None: + conditions = [] + if true_works is None: + true_works = [] + if false_works is None: + false_works = [] + if conditions and type(conditions) not in [tuple, list]: + conditions = [conditions] + if true_works and type(true_works) not in [tuple, list]: + true_works = [true_works] + if false_works and type(false_works) not in [tuple, list]: + false_works = [false_works] + self.validate_conditions(conditions) + + self.operator = operator + self.conditions = [] + self.true_works = [] + self.false_works = [] + + self.conditions = conditions + self.true_works = true_works + self.false_works = false_works + + def get_class_name(self): + return self.__class__.__name__ + + def get_internal_id(self): + return self.internal_id + + def get_template_id(self): + return self.template_id + + def copy(self): + new_cond = copy.deepcopy(self) + return new_cond + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + @property + def conditions(self): + # return self.get_metadata_item('true_works', []) + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + + @property + def true_works(self): + # return self.get_metadata_item('true_works', []) + return self._true_works + + @true_works.setter + def true_works(self, value): + self._true_works = value + true_work_meta = self.get_metadata_item('true_works', {}) + for work in value: + if work is None: + continue + if isinstance(work, Work): + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, CompositeCondition): + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, Workflow): + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + self.add_metadata_item('true_works', true_work_meta) + + @property + def false_works(self): + # return self.get_metadata_item('false_works', []) + return self._false_works + + @false_works.setter + def false_works(self, value): + self._false_works = value + false_work_meta = self.get_metadata_item('false_works', {}) + for work in value: + if work is None: + continue + if isinstance(work, Work): + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, CompositeCondition): + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, Workflow): + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + self.add_metadata_item('false_works', false_work_meta) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + for cond in conditions: + assert(inspect.ismethod(cond)) + + def add_condition(self, cond): + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + + # self.conditions.append({'condition': cond, 'current_work': cond.__self__}) + + self._conditions.append(cond) + + def load_metadata(self): + # conditions = self.get_metadata_item('conditions', []) + # true_works_meta = self.get_metadata_item('true_works', {}) + # false_works_meta = self.get_metadata_item('false_works', {}) + pass + + def to_dict(self): + # print('to_dict') + ret = {'class': self.__class__.__name__, + 'module': self.__class__.__module__, + 'attributes': {}} + for key, value in self.__dict__.items(): + # print(key) + # print(value) + # if not key.startswith('__') and not key.startswith('_'): + if not key.startswith('__'): + if key == 'logger': + value = None + elif key == '_conditions': + new_value = [] + for cond in value: + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id()} + else: + new_cond = cond + new_value.append(new_cond) + value = new_value + elif key in ['_true_works', '_false_works']: + new_value = [] + for w in value: + if isinstance(w, Work): + new_w = w.get_internal_id() + elif isinstance(w, CompositeCondition): + new_w = w.to_dict() + elif isinstance(w, Workflow): + new_w = w.to_dict() + else: + new_w = w + new_value.append(new_w) + value = new_value + else: + value = self.to_dict_l(value) + ret['attributes'][key] = value + return ret + + def get_work_from_id(self, work_id, works): + return works[work_id] + + def load_conditions(self, works): + new_conditions = [] + for cond in self.conditions: + if callable(cond): + new_conditions.append(cond) + else: + if 'idds_method' in cond and 'idds_method_internal_id' in cond: + internal_id = cond['idds_method_internal_id'] + work = self.get_work_from_id(internal_id, works) + if work is not None: + new_cond = getattr(work, cond['idds_method']) + else: + self.logger.error("Work cannot be found for %s" % (internal_id)) + new_cond = cond + else: + new_cond = cond + new_conditions.append(new_cond) + self.conditions = new_conditions + + new_true_works = [] + for w in self.true_works: + if isinstance(w, CompositeCondition) or isinstance(w, Workflow): + # work = w.load_conditions(works, works_template) + w.load_conditions(works) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, works) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_true_works.append(work) + self.true_works = new_true_works + + new_false_works = [] + for w in self.false_works: + if isinstance(w, CompositeCondition) or isinstance(w, Workflow): + # work = w.load_condtions(works, works_template) + w.load_conditions(works) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, works) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_false_works.append(work) + self.false_works = new_false_works + + def all_works(self): + works = [] + works = works + self.all_pre_works() + works = works + self.all_next_works() + return works + + def all_condition_ids(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__.get_internal_id()) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_condition_ids() + return works + + def all_pre_works(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_pre_works() + return works + + def all_next_works(self): + works = [] + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_next_works() + else: + works.append(work) + return works + + def get_current_cond_status(self, cond): + if callable(cond): + if cond(): + return True + else: + return False + else: + if cond: + return True + else: + return False + + def get_cond_status(self): + if self.operator == ConditionOperator.And: + for cond in self.conditions: + if not self.get_current_cond_status(cond): + return False + return True + else: + for cond in self.conditions: + if self.get_current_cond_status(cond): + return True + return False + + def get_condition_status(self): + return self.get_cond_status() + + def is_condition_true(self): + if self.get_cond_status(): + return True + return False + + def is_condition_false(self): + if not self.get_cond_status(): + return True + return False + + def get_next_works(self, trigger=ConditionTrigger.NotTriggered): + works = [] + if self.get_cond_status(): + true_work_meta = self.get_metadata_item('true_works', {}) + for work in self.true_works: + if isinstance(work, CompositeCondition): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + if trigger == ConditionTrigger.ToTrigger: + if not true_work_meta[work.get_internal_id()]['triggered']: + true_work_meta[work.get_internal_id()]['triggered'] = True + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not true_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if true_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + self.add_metadata_item('true_works', true_work_meta) + else: + false_work_meta = self.get_metadata_item('false_works', {}) + for work in self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + if trigger == ConditionTrigger.ToTrigger: + if not false_work_meta[work.get_internal_id()]['triggered']: + false_work_meta[work.get_internal_id()]['triggered'] = True + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not false_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if false_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + self.add_metadata_item('false_works', false_work_meta) + return works + + +class AndCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(AndCondition, self).__init__(operator=ConditionOperator.And, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class OrCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(OrCondition, self).__init__(operator=ConditionOperator.Or, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class Condition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + super(Condition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + # to support load from old conditions + @property + def cond(self): + # return self.get_metadata_item('true_works', []) + return self.conditions[0] if len(self.conditions) >= 1 else None + + @cond.setter + def cond(self, value): + self.conditions = [value] + + @property + def true_work(self): + # return self.get_metadata_item('true_works', []) + return self.true_works if len(self.true_works) >= 1 else None + + @true_work.setter + def true_work(self, value): + self.true_works = [value] + + @property + def false_work(self): + # return self.get_metadata_item('true_works', []) + return self.false_works if len(self.false_works) >= 1 else None + + @false_work.setter + def false_work(self, value): + self.false_works = [value] + + +class TemplateCondition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + if true_work is not None and not isinstance(true_work, Work): + raise exceptions.IDDSException("true_work can only be set with Work class") + if false_work is not None and not isinstance(false_work, Work): + raise exceptions.IDDSException("false_work can only be set with Work class") + + super(TemplateCondition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + if len(conditions) > 1: + raise exceptions.IDDSException("Condition class can only support one condition. To support multiple condition, please use CompositeCondition.") + for cond in conditions: + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + + def add_condition(self, cond): + raise exceptions.IDDSException("Condition class doesn't support add_condition. To support multiple condition, please use CompositeCondition.") + + +class ParameterLink(Base): + def __init__(self, parameters): + self.parameters = parameters + self.internal_id = str(uuid.uuid4())[:8] + self.template_id = self.internal_id + + def get_internal_id(self): + return self.internal_id + + def get_parameter_value(self, work, p): + p_f = getattr(work, p, 'None') + if p_f: + if callable(p_f): + return p_f() + else: + return p_f + else: + return None + + def set_parameters(self, work): + p_values = {} + for p in self.parameters: + p_values[p] = self.get_parameter_value(work, p) + self.add_metadata_item('parameters', p_values) + + def get_parameters(self): + return self.get_metadata_item('parameters', {}) + + +class WorkflowBase(Base): + + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + """ + Init a workflow. + """ + self._works = {} + self._conditions = {} + self._work_conds = {} + + self.parameter_links = {} + self.parameter_links_source = {} + self.parameter_links_destination = {} + + super(WorkflowBase, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_work_id = self.internal_id + # self.template_work_id = str(uuid.uuid4())[:8] + self.lifetime = lifetime + self.pending_time = pending_time + + if name: + self._name = name + "." + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + else: + self._name = 'idds.workflow.' + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + + if workload_id is None: + workload_id = int(time.time()) + self.workload_id = workload_id + + self.logger = logger + if self.logger is None: + self.setup_logger() + + self._works = {} + self.works = {} + self.work_sequence = {} # order list + + self.terminated_works = [] + self.initial_works = [] + # if the primary initial_work is not set, it's the first initial work. + self.primary_initial_work = None + self.independent_works = [] + + self.first_initial = False + self.new_to_run_works = [] + self.current_running_works = [] + + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + self.num_total_works = 0 + + self.last_work = None + + self.last_updated_at = datetime.datetime.utcnow() + self.expired = False + + self.to_update_transforms = {} + + # user defined Condition class + self.user_defined_conditions = {} + + self.username = None + self.userdn = None + self.proxy = None + + self._loop_condition_position = 'end' + self.loop_condition = None + + """ + self._running_data_names = [] + for name in ['internal_id', 'template_work_id', 'workload_id', 'work_sequence', 'terminated_works', + 'first_initial', 'new_to_run_works', 'current_running_works', + 'num_subfinished_works', 'num_finished_works', 'num_failed_works', 'num_cancelled_works', 'num_suspended_works', + 'num_expired_works', 'num_total_works', 'last_work']: + self._running_data_names.append(name) + for name in ['works']: + self._running_data_names.append(name) + """ + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + + def get_template_work_id(self): + return self.template_work_id + + def get_template_id(self): + return self.template_work_id + + @property + def workload_id(self): + return self.get_metadata_item('workload_id') + + @workload_id.setter + def workload_id(self, value): + self.add_metadata_item('workload_id', value) + + @property + def lifetime(self): + # return self.get_metadata_item('lifetime', None) + return getattr(self, '_lifetime', None) + + @lifetime.setter + def lifetime(self, value): + # self.add_metadata_item('lifetime', value) + self._lifetime = value + + @property + def pending_time(self): + # return self.get_metadata_item('pending_time', None) + return getattr(self, '_pending_time', None) + + @pending_time.setter + def pending_time(self, value): + # self.add_metadata_item('pending_time', value) + self._pending_time = value + + @property + def last_updated_at(self): + last_updated_at = self.get_metadata_item('last_updated_at', None) + if last_updated_at and type(last_updated_at) in [str]: + last_updated_at = str_to_date(last_updated_at) + return last_updated_at + + @last_updated_at.setter + def last_updated_at(self, value): + self.add_metadata_item('last_updated_at', value) + + def has_new_updates(self): + self.last_updated_at = datetime.datetime.utcnow() + + @property + def expired(self): + t = self.get_metadata_item('expired', False) + if type(t) in [bool]: + return t + elif type(t) in [str] and t.lower() in ['true']: + return True + else: + return False + + @expired.setter + def expired(self, value): + self.add_metadata_item('expired', value) + + @property + def works(self): + return self._works + + @works.setter + def works(self, value): + self._works = value + work_metadata = {} + if self._works: + for k in self._works: + work = self._works[k] + if isinstance(work, Workflow): + work_metadata[k] = {'type': 'workflow', + 'metadata': work.metadata} + else: + work_metadata[k] = {'type': 'work', + 'work_id': work.work_id, + 'workload_id': work.workload_id, + 'status': work.status, + 'substatus': work.substatus, + 'transforming': work.transforming} + self.add_metadata_item('works', work_metadata) + + def refresh_works(self): + work_metadata = {} + if self._works: + for k in self._works: + work = self._works[k] + if isinstance(work, Workflow): + work.refresh_works() + work_metadata[k] = {'type': 'workflow', + 'metadata': work.metadata} + else: + work_metadata[k] = {'type': 'work', + 'work_id': work.work_id, + 'workload_id': work.workload_id, + 'status': work.status, + 'substatus': work.substatus, + 'transforming': work.transforming} + if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): + self.last_updated_at = work.last_updated_at + self.add_metadata_item('works', work_metadata) + + def load_works(self): + work_metadata = self.get_metadata_item('works', {}) + for k in self._works: + if k in work_metadata: + if work_metadata[k]['type'] == 'work': + self._works[k].work_id = work_metadata[k]['work_id'] + self._works[k].workload_id = work_metadata[k]['workload_id'] + self._works[k].transforming = work_metadata[k]['transforming'] + self._works[k].status = work_metadata[k]['status'] + self._works[k].substatus = work_metadata[k]['substatus'] + elif work_metadata[k]['type'] == 'workflow': + self._works[k].metadata = work_metadata[k]['metadata'] + + work = self._works[k] + if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): + self.last_updated_at = work.last_updated_at + + @property + def conditions(self): + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + conditions_metadata = {} + if self._conditions: + for k in self._conditions: + conditions_metadata[k] = self._conditions[k].metadata + self.add_metadata_item('conditions', conditions_metadata) + + @property + def work_conds(self): + return self._work_conds + + @work_conds.setter + def work_conds(self, value): + self._work_conds = value + # self.add_metadata_item('work_conds', value) + + def load_work_conditions(self): + conditions_metadata = self.get_metadata_item('conditions', {}) + for cond_internal_id in self._conditions: + if cond_internal_id in conditions_metadata: + self.conditions[cond_internal_id].metadata = conditions_metadata[cond_internal_id] + self.conditions[cond_internal_id].load_conditions(self.works) + + # work_conds = self.get_metadata_item('work_conds', {}) + # self._work_conds = work_conds + + @property + def loop_condition(self): + return self._loop_condition + + @loop_condition.setter + def loop_condition(self, value): + # self._loop_condition_position = position + self._loop_condition = value + if self._loop_condition: + self.add_metadata_item('loop_condition', self._loop_condition.get_condition_status()) + + @property + def work_sequence(self): + return self.get_metadata_item('work_sequence', {}) + + @work_sequence.setter + def work_sequence(self, value): + self.add_metadata_item('work_sequence', value) + + @property + def terminated_works(self): + return self.get_metadata_item('terminated_works', []) + + @terminated_works.setter + def terminated_works(self, value): + self.add_metadata_item('terminated_works', value) + + @property + def first_initial(self): + return self.get_metadata_item('first_initial', False) + + @first_initial.setter + def first_initial(self, value): + self.add_metadata_item('first_initial', value) + + @property + def new_to_run_works(self): + return self.get_metadata_item('new_to_run_works', []) + + @new_to_run_works.setter + def new_to_run_works(self, value): + self.add_metadata_item('new_to_run_works', value) + + @property + def current_running_works(self): + return self.get_metadata_item('current_running_works', []) + + @current_running_works.setter + def current_running_works(self, value): + self.add_metadata_item('current_running_works', value) + + @property + def num_subfinished_works(self): + return self.get_metadata_item('num_subfinished_works', 0) + + @num_subfinished_works.setter + def num_subfinished_works(self, value): + self.add_metadata_item('num_subfinished_works', value) + + @property + def num_finished_works(self): + return self.get_metadata_item('num_finished_works', 0) + + @num_finished_works.setter + def num_finished_works(self, value): + self.add_metadata_item('num_finished_works', value) + + @property + def num_failed_works(self): + return self.get_metadata_item('num_failed_works', 0) + + @num_failed_works.setter + def num_failed_works(self, value): + self.add_metadata_item('num_failed_works', value) + + @property + def num_cancelled_works(self): + return self.get_metadata_item('num_cancelled_works', 0) + + @num_cancelled_works.setter + def num_cancelled_works(self, value): + self.add_metadata_item('num_cancelled_works', value) + + @property + def num_suspended_works(self): + return self.get_metadata_item('num_suspended_works', 0) + + @num_suspended_works.setter + def num_suspended_works(self, value): + self.add_metadata_item('num_suspended_works', value) + + @property + def num_expired_works(self): + return self.get_metadata_item('num_expired_works', 0) + + @num_expired_works.setter + def num_expired_works(self, value): + self.add_metadata_item('num_expired_works', value) + + @property + def num_total_works(self): + return self.get_metadata_item('num_total_works', 0) + + @num_total_works.setter + def num_total_works(self, value): + self.add_metadata_item('num_total_works', value) + + @property + def last_work(self): + return self.get_metadata_item('last_work', None) + + @last_work.setter + def last_work(self, value): + self.add_metadata_item('last_work', value) + + @property + def to_update_transforms(self): + return self.get_metadata_item('to_update_transforms', {}) + + @to_update_transforms.setter + def to_update_transforms(self, value): + self.add_metadata_item('to_update_transforms', value) + + def load_metadata(self): + self.load_works() + self.load_work_conditions() + self.load_parameter_links() + + def get_class_name(self): + return self.__class__.__name__ + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) + + def log_info(self, info): + if self.logger is None: + self.setup_logger() + self.logger.info(info) + + def log_debug(self, info): + if self.logger is None: + self.setup_logger() + self.logger.debug(info) + + def get_internal_id(self): + return self.internal_id + + def copy(self): + new_wf = copy.deepcopy(self) + return new_wf + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + def get_works(self): + return self.works + + def get_new_work_to_run(self, work_id, new_parameters=None): + # 1. initialize works + # template_id = work.get_template_id() + work = self.works[work_id] + if isinstance(work, Workflow): + work.sync_works() + + work.sequence_id = self.num_total_works + + works = self.works + self.works = works + # self.work_sequence.append(new_work.get_internal_id()) + self.work_sequence[str(self.num_total_works)] = work.get_internal_id() + self.num_total_works += 1 + self.new_to_run_works.append(work.get_internal_id()) + self.last_work = work.get_internal_id() + else: + if new_parameters: + work.set_parameters(new_parameters) + work.sequence_id = self.num_total_works + + work.initialize_work() + works = self.works + self.works = works + # self.work_sequence.append(new_work.get_internal_id()) + self.work_sequence[str(self.num_total_works)] = work.get_internal_id() + self.num_total_works += 1 + self.new_to_run_works.append(work.get_internal_id()) + self.last_work = work.get_internal_id() + + return work + + def register_user_defined_condition(self, condition): + cond_src = inspect.getsource(condition) + self.user_defined_conditions[condition.__name__] = cond_src + + def load_user_defined_condition(self): + # try: + # Condition() + # except NameError: + # global Condition + # import Condition + + for cond_src_name in self.user_defined_conditions: + # global cond_src_name + exec(self.user_defined_conditions[cond_src_name]) + + def set_workload_id(self, workload_id): + self.workload_id = workload_id + + def get_workload_id(self): + return self.workload_id + + def add_initial_works(self, work): + self.initial_works.append(work.get_internal_id()) + if self.primary_initial_work is None: + self.primary_initial_work = work.get_internal_id() + + def add_work(self, work, initial=False, primary=False): + self.first_initial = False + self.works[work.get_internal_id()] = work + if initial: + if primary: + self.primary_initial_work = work.get_internal_id() + self.add_initial_works(work) + + self.independent_works.append(work.get_internal_id()) + + def add_condition(self, cond): + self.first_initial = False + cond_works = cond.all_works() + for cond_work in cond_works: + assert(cond_work.get_internal_id() in self.get_works()) + + conditions = self.conditions + conditions[cond.get_internal_id()] = cond + self.conditions = conditions + + # if cond.current_work not in self.work_conds: + # self.work_conds[cond.current_work] = [] + # self.work_conds[cond.current_work].append(cond) + work_conds = self.work_conds + for work in cond.all_pre_works(): + if work.get_internal_id() not in work_conds: + work_conds[work.get_internal_id()] = [] + work_conds[work.get_internal_id()].append(cond.get_internal_id()) + self.work_conds = work_conds + + # if a work is a true_work or false_work of a condition, + # should remove it from independent_works + cond_next_works = cond.all_next_works() + for next_work in cond_next_works: + if next_work.get_internal_id() in self.independent_works: + self.independent_works.remove(next_work.get_internal_id()) + + def add_parameter_link(self, work_source, work_destinations, parameter_link): + self.parameter_links[parameter_link.get_internal_id()] = parameter_link + if work_source.get_internal_id() not in self.parameter_links_source: + self.parameter_links_source[work_source.get_internal_id()] = [] + self.parameter_links_source[work_source.get_internal_id()].append(parameter_link.get_internal_id()) + + if type(work_destinations) not in [list, tuple]: + work_destinations = [] + for work_destination in work_destinations: + if work_destination.get_internal_id() not in self.parameter_links_destination: + self.parameter_links_destination[work_destination.get_internal_id()] = [] + self.parameter_links_destination[work_destination.get_internal_id()].append(parameter_link.get_internal_id()) + + def set_source_parameters(self, internal_id): + work = self.works[internal_id] + p_metadata = {} + if internal_id in self.parameter_links_source: + for p_id in self.parameter_links_source[internal_id]: + p_link = self.parameter_links[p_id] + p_link.set_parameters(work) + p_metadata[p_id] = p_link.metadata + self.add_metadata_item('parameter_links', p_metadata) + + def get_destination_parameters(self, internal_id): + # work = self.works[internal_id] + parameters = {} + if internal_id in self.parameter_links_destination: + for p_id in self.parameter_links_destination[internal_id]: + p_link = self.parameter_links[p_id] + parameters.update(p_link.get_parameters()) + return parameters + + def load_parameter_links(self): + p_metadata = self.get_metadata_item('parameter_links', {}) + for p_id in self.parameter_links: + if p_id in p_metadata: + self.parameter_links[p_id].metadata = p_metadata[p_id] + + def enable_next_works(self, work, cond): + self.log_debug("Checking Work %s condition: %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + # load_conditions should cover it. + # if cond and self.is_class_method(cond.cond): + # # cond_work_id = self.works[cond.cond['idds_method_class_id']] + # cond.cond = getattr(work, cond.cond['idds_method']) + + self.log_info("Work %s condition: %s" % (work.get_internal_id(), cond.conditions)) + next_works = cond.get_next_works(trigger=ConditionTrigger.ToTrigger) + self.log_info("Work %s condition status %s" % (work.get_internal_id(), cond.get_cond_status())) + self.log_info("Work %s next works %s" % (work.get_internal_id(), str(next_works))) + new_next_works = [] + if next_works is not None: + for next_work in next_works: + parameters = self.get_destination_parameters(next_work.get_internal_id()) + new_next_work = self.get_new_work_to_run(next_work.get_internal_id(), parameters) + work.add_next_work(new_next_work.get_internal_id()) + # cond.add_condition_work(new_next_work) ####### TODO: + new_next_works.append(new_next_work) + return new_next_works + + def add_loop_condition(self, condition, position='end'): + self.loop_condition_position = position + self.loop_condition = condition + + def has_loop_condition(self): + if self.loop_condition: + return True + return False + + def get_loop_condition_status(self): + if self.has_loop_condition(): + self.loop_condition.load_conditions(self.works) + return self.loop_condition.get_condition_status() + return False + + def __str__(self): + return str(json_dumps(self)) + + def get_new_works(self): + """ + *** Function called by Marshaller agent. + + new works to be ready to start + """ + self.sync_works() + works = [] + for k in self.new_to_run_works: + if isinstance(self.works[k], Work): + works.append(self.works[k]) + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_new_works() + for k in self.current_running_works: + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_new_works() + return works + + def get_current_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + works = [] + for k in self.current_running_works: + if isinstance(self.works[k], Work): + works.append(self.works[k]) + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_current_works() + return works + + def get_all_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + + works = [] + for k in self.works: + if isinstance(self.works[k], Work): + works.append(self.works[k]) + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_all_works() + return works + + def get_primary_initial_collection(self): + """ + *** Function called by Clerk agent. + """ + + if self.primary_initial_work: + return self.get_works()[self.primary_initial_work].get_primary_input_collection() + elif self.initial_works: + return self.get_works()[self.initial_works[0]].get_primary_input_collection() + elif self.independent_works: + return self.get_works()[self.independent_works[0]].get_primary_input_collection() + else: + keys = self.get_works().keys() + return self.get_works()[keys[0]].get_primary_input_collection() + return None + + def get_dependency_works(self, work_id, depth, max_depth): + if depth > max_depth: + return [] + + deps = [] + for dep_work_id in self.work_dependencies[work_id]: + deps.append(dep_work_id) + l_deps = self.get_dependency_works(dep_work_id, depth + 1, max_depth) + deps += l_deps + deps = list(dict.fromkeys(deps)) + return deps + + def order_independent_works(self): + ind_work_ids = self.independent_works + self.independent_works = [] + self.work_dependencies = {} + for ind_work_id in ind_work_ids: + work = self.works[ind_work_id] + self.work_dependencies[ind_work_id] = [] + for ind_work_id1 in ind_work_ids: + if ind_work_id == ind_work_id1: + continue + work1 = self.works[ind_work_id1] + if work.depend_on(work1): + self.work_dependencies[ind_work_id].append(ind_work_id1) + self.log_debug('work dependencies 1: %s' % str(self.work_dependencies)) + + max_depth = len(ind_work_ids) + 1 + work_dependencies = copy.deepcopy(self.work_dependencies) + for work_id in work_dependencies: + deps = self.get_dependency_works(work_id, 0, max_depth) + self.work_dependencies[work_id] = deps + self.log_debug('work dependencies 2: %s' % str(self.work_dependencies)) + + while True: + for work_id in self.work_dependencies: + if work_id not in self.independent_works and len(self.work_dependencies[work_id]) == 0: + self.independent_works.append(work_id) + for work_id in self.independent_works: + if work_id in self.work_dependencies: + del self.work_dependencies[work_id] + for work_id in self.work_dependencies: + for in_work_id in self.independent_works: + if in_work_id in self.work_dependencies[work_id]: + self.work_dependencies[work_id].remove(in_work_id) + if not self.work_dependencies: + break + self.log_debug('independent_works: %s' % str(self.independent_works)) + + def first_initialize(self): + # set new_to_run works + if not self.first_initial: + self.first_initial = True + self.order_independent_works() + if self.initial_works: + tostart_works = self.initial_works + elif self.independent_works: + tostart_works = self.independent_works + else: + tostart_works = list(self.get_works().keys()) + tostart_works = [tostart_works[0]] + + for work_id in tostart_works: + self.get_new_work_to_run(work_id) + + def sync_works(self): + self.first_initialize() + + self.refresh_works() + + for k in self.works: + work = self.works[k] + self.log_debug("work %s is_terminated(%s:%s)" % (work.get_internal_id(), work.is_terminated(), work.get_status())) + + for work in [self.works[k] for k in self.new_to_run_works]: + if work.transforming: + self.new_to_run_works.remove(work.get_internal_id()) + self.current_running_works.append(work.get_internal_id()) + + for work in [self.works[k] for k in self.current_running_works]: + if isinstance(work, Workflow): + work.sync_works() + + if work.is_terminated(): + self.set_source_parameters(work.get_internal_id()) + + if work.get_internal_id() in self.work_conds: + self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + json_dumps(self.work_conds[work.get_internal_id()], sort_keys=True, indent=4))) + for cond_id in self.work_conds[work.get_internal_id()]: + cond = self.conditions[cond_id] + self.log_debug("Work %s has condition dependencie %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + self.enable_next_works(work, cond) + + if work.is_terminated(): + self.log_info("Work %s is terminated(%s)" % (work.get_internal_id(), work.get_status())) + self.log_debug("Work conditions: %s" % json_dumps(self.work_conds, sort_keys=True, indent=4)) + if work.get_internal_id() not in self.work_conds: + # has no next work + self.log_info("Work %s has no condition dependencies" % work.get_internal_id()) + self.terminated_works.append(work.get_internal_id()) + self.current_running_works.remove(work.get_internal_id()) + else: + # self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + # json_dumps(self.work_conds[work.get_template_id()], sort_keys=True, indent=4))) + # for cond in self.work_conds[work.get_template_id()]: + # self.enable_next_works(work, cond) + self.terminated_works.append(work.get_internal_id()) + self.current_running_works.remove(work.get_internal_id()) + + if work.is_finished(): + self.num_finished_works += 1 + elif work.is_subfinished(): + self.num_subfinished_works += 1 + elif work.is_failed(): + self.num_failed_works += 1 + elif work.is_expired(): + self.num_expired_works += 1 + elif work.is_cancelled(): + self.num_cancelled_works += 1 + elif work.is_suspended(): + self.num_suspended_works += 1 + + # if work.is_terminated(): + # # if it's a loop workflow, to generate new loop + # if isinstance(work, Workflow): + # work.sync_works() + log_str = "num_total_works: %s" % self.num_total_works + log_str += ", num_finished_works: %s" % self.num_finished_works + log_str += ", num_subfinished_works: %s" % self.num_subfinished_works + log_str += ", num_failed_works: %s" % self.num_failed_works + log_str += ", num_expired_works: %s" % self.num_expired_works + log_str += ", num_cancelled_works: %s" % self.num_cancelled_works + log_str += ", num_suspended_works: %s" % self.num_suspended_works + self.log_debug(log_str) + + def resume_works(self): + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + + self.last_updated_at = datetime.datetime.utcnow() + + t_works = self.terminated_works + self.terminated_works = [] + self.current_running_works = self.current_running_works + t_works + for work in [self.works[k] for k in self.current_running_works]: + if isinstance(work, Workflow): + work.resume_works() + else: + work.resume_work() + + def clean_works(self): + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + self.num_total_works = 0 + + self.last_updated_at = datetime.datetime.utcnow() + + self.terminated_works = [] + self.current_running_works = [] + self.works = {} + self.work_sequence = {} # order list + + self.first_initial = False + self.new_to_run_works = [] + + def get_exact_workflows(self): + """ + *** Function called by Clerk agent. + + TODO: The primary dataset for the initial work is a dataset with '*'. + workflow.primary_initial_collection = 'some datasets with *' + collections = get_collection(workflow.primary_initial_collection) + wfs = [] + for coll in collections: + wf = self.copy() + wf.name = self.name + "_" + number + wf.primary_initial_collection = coll + wfs.append(wf) + return wfs + """ + return [self] + + def is_terminated(self): + """ + *** Function called by Marshaller agent. + """ + self.sync_works() + if len(self.new_to_run_works) == 0 and len(self.current_running_works) == 0: + return True + return False + + def is_finished(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and self.num_finished_works == self.num_total_works + + def is_subfinished(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_finished_works + self.num_subfinished_works > 0 and self.num_finished_works + self.num_subfinished_works <= self.num_total_works) + + def is_failed(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_failed_works > 0) and (self.num_cancelled_works == 0) and (self.num_suspended_works == 0) and (self.num_expired_works == 0) + + def is_to_expire(self, expired_at=None, pending_time=None, request_id=None): + if self.expired: + # it's already expired. avoid sending duplicated messages again and again. + return False + if expired_at: + if type(expired_at) in [str]: + expired_at = str_to_date(expired_at) + if expired_at < datetime.datetime.utcnow(): + self.logger.info("Request(%s) expired_at(%s) is smaller than utc now(%s), expiring" % (request_id, + expired_at, + datetime.datetime.utcnow())) + return True + + act_pending_time = None + if self.pending_time: + # in days + act_pending_time = float(self.pending_time) + else: + if pending_time: + act_pending_time = float(pending_time) + if act_pending_time: + act_pending_seconds = int(86400 * act_pending_time) + if self.last_updated_at + datetime.timedelta(seconds=act_pending_seconds) < datetime.datetime.utcnow(): + log_str = "Request(%s) last updated at(%s) + pending seconds(%s)" % (request_id, + self.last_updated_at, + act_pending_seconds) + log_str += " is smaller than utc now(%s), expiring" % (datetime.datetime.utcnow()) + self.logger.info(log_str) + return True + + return False + + def is_expired(self): + """ + *** Function called by Marshaller agent. + """ + # return self.is_terminated() and (self.num_expired_works > 0) + return self.is_terminated() and self.expired + + def is_cancelled(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_cancelled_works > 0) + + def is_suspended(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_suspended_works > 0) + + def get_terminated_msg(self): + """ + *** Function called by Marshaller agent. + """ + if self.last_work: + return self.works[self.last_work].get_terminated_msg() + return None + + def get_status(self): + if self.is_terminated(): + if self.is_finished(): + return WorkStatus.Finished + elif self.is_subfinished(): + return WorkStatus.SubFinished + elif self.is_failed(): + return WorkStatus.Failed + elif self.is_expired(): + return WorkStatus.Expired + elif self.is_cancelled(): + return WorkStatus.Cancelled + elif self.is_suspended(): + return WorkStatus.Suspended + return WorkStatus.Transforming + + def depend_on(self, work): + return False + + def add_proxy(self): + self.proxy = get_proxy() + if not self.proxy: + raise Exception("Cannot get local proxy") + + def get_proxy(self): + return self.proxy + + +class Workflow(Base): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # super(Workflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + self.logger = logger + if self.logger is None: + self.setup_logger() + + self.template = WorkflowBase(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + self.num_run = 0 + self.runs = {} + self.loop_condition_position = 'end' + + def setup_logger(self): + # Setup logger + self.logger = logging.getLogger(self.get_class_name()) + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + @property + def metadata(self): + run_metadata = {'num_run': self.num_run, + 'runs': {}} + for run_id in self.runs: + run_metadata['runs'][run_id] = self.runs[run_id].metadata + return run_metadata + + @metadata.setter + def metadata(self, value): + run_metadata = value + self.num_run = run_metadata['num_run'] + runs = run_metadata['runs'] + for run_id in runs: + self.runs[run_id] = self.template.copy() + self.runs[run_id].metadata = runs[run_id] + # self.add_metadata_item('runs', ) + + @property + def independent_works(self): + if self.runs: + return self.runs[str(self.num_run)].independent_works + return self.template.independent_works + + @independent_works.setter + def independent_works(self, value): + if self.runs: + self.runs[str(self.num_run)].independent_works = value + self.template.independent_works = value + + @property + def last_updated_at(self): + if self.runs: + return self.runs[str(self.num_run)].last_updated_at + return None + + @last_updated_at.setter + def last_updated_at(self, value): + if self.runs: + self.runs[str(self.num_run)].last_updated_at = value + + @property + def transforming(self): + if self.runs and str(self.num_run) in self.runs: + return True + return False + + @transforming.setter + def transforming(self, value): + if self.num_run < 1: + self.num_run = 1 + if str(self.num_run) not in self.runs: + self.runs[str(self.num_run)] = self.template.copy() + + def set_workload_id(self, workload_id): + if self.runs: + self.runs[str(self.num_run)].workload_id = workload_id + else: + self.template.workload_id = workload_id + # self.dynamic.workload_id = workload_id + + def get_internal_id(self): + if self.runs: + return self.runs[str(self.num_run)].get_internal_id() + return self.template.get_internal_id() + + def get_workload_id(self): + if self.runs: + return self.runs[str(self.num_run)].workload_id + return self.template.workload_id + + def add_work(self, work, initial=False, primary=False): + self.template.add_work(work, initial, primary) + + def add_condition(self, cond): + self.template.add_condition(cond) + + def get_new_works(self): + self.sync_works() + if self.runs: + return self.runs[str(self.num_run)].get_new_works() + return [] + + def get_current_works(self): + self.sync_works() + if self.runs: + return self.runs[str(self.num_run)].get_current_works() + return [] + + def get_all_works(self): + self.sync_works() + if self.runs: + return self.runs[str(self.num_run)].get_all_works() + return [] + + def get_primary_initial_collection(self): + if self.runs: + return self.runs[str(self.num_run)].get_primary_initial_collection() + return self.template.get_primary_initial_collection() + + def resume_works(self): + if self.runs: + self.runs[str(self.num_run)].resume_works() + + def clean_works(self): + if self.runs: + self.runs[str(self.num_run)].clean_works() + + def is_terminated(self): + if self.runs: + if self.runs[str(self.num_run)].is_terminated(): + if not self.runs[str(self.num_run)].has_loop_condition() or not self.runs[str(self.num_run)].get_loop_condition_status(): + return True + return False + + def is_finished(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_finished() + return False + + def is_subfinished(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_subfinished() + return False + + def is_failed(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_failed() + return False + + def is_expired(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_expired() + return False + + def is_cancelled(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_cancelled() + return False + + def is_suspended(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_suspended() + return False + + def get_terminated_msg(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].get_terminated_msg() + return None + + def get_status(self): + if not self.runs: + return WorkStatus.New + if not self.is_terminated(): + return WorkStatus.Transforming + return self.runs[str(self.num_run)].get_status() + + def depend_on(self, work): + return self.template.depend_on(work) + + def add_proxy(self): + self.template.add_proxy() + + def get_proxy(self): + self.template.get_proxy() + + def add_loop_condition(self, condition, position='end'): + if not position or position != 'begin': + position = 'end' + position = 'end' # force position to end currently. position = 'begin' is not supported now. + self.template.add_loop_condition(condition, position=position) + self.loop_condition_position = position + + def refresh_works(self): + if self.runs: + self.runs[str(self.num_run)].refresh_works() + + def sync_works(self): + # position is end. + if self.num_run < 1: + self.num_run = 1 + if str(self.num_run) not in self.runs: + self.runs[str(self.num_run)] = self.template.copy() + + self.runs[str(self.num_run)].sync_works() + + if self.runs[str(self.num_run)].is_terminated(): + if self.runs[str(self.num_run)].has_loop_condition(): + if self.runs[str(self.num_run)].get_loop_condition_status(): + self.num_run += 1 + self.runs[str(self.num_run)] = self.template.copy() + + +class SubWorkflow(Workflow): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # Init a workflow. + super(SubWorkflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + + +class LoopWorkflow(Workflow): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # Init a workflow. + super(LoopWorkflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) diff --git a/workflow/lib/idds/workflowv2/__init__.py b/workflow/lib/idds/workflowv2/__init__.py new file mode 100644 index 00000000..865b774e --- /dev/null +++ b/workflow/lib/idds/workflowv2/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2019 diff --git a/workflow/lib/idds/workflowv2/base.py b/workflow/lib/idds/workflowv2/base.py new file mode 100644 index 00000000..24fd2198 --- /dev/null +++ b/workflow/lib/idds/workflowv2/base.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + +import logging +import pickle +import urllib +# from enum import Enum + +from idds.common.dict_class import DictClass + + +class IDDSDict(dict): + def __setitem__(self, key, value): + if key == 'test': + pass + else: + super().__setitem__(key, value) + + +class IDDSMetadata(DictClass): + def __init__(self): + pass + + def add_item(self, key, value): + setattr(self, key, value) + + def get_item(self, key, default): + return getattr(self, key, default) + + +class Base(DictClass): + def __init__(self): + self.metadata = IDDSMetadata() + pass + + def add_metadata_item(self, key, value): + self.metadata.add_item(key, value) + + def get_metadata_item(self, key, default=None): + return self.metadata.get_item(key, default) + + def load_metadata(self): + pass + + @property + def metadata(self): + return self._metadata + + @metadata.setter + def metadata(self, value): + self._metadata = value + self.load_metadata() + + def IDDSProperty(self, attribute): + def _get(self, attribute): + self.get_metadata_item(attribute, None) + + def _set(self, attribute, value): + self.add_metadata_item(attribute, value) + + attribute = property(_get, _set) + return attribute + + def serialize(self): + return urllib.parse.quote_from_bytes(pickle.dumps(self)) + + @staticmethod + def deserialize(obj): + # return urllib.parse.unquote_to_bytes(pickle.loads(obj)) + return pickle.loads(urllib.parse.unquote_to_bytes(obj)) + + def get_class_name(self): + return self.__class__.__name__ + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) diff --git a/workflow/lib/idds/workflowv2/version.py b/workflow/lib/idds/workflowv2/version.py new file mode 100644 index 00000000..a09efc7f --- /dev/null +++ b/workflow/lib/idds/workflowv2/version.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2019 - 2021 + + +release_version = "0.7.7" diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py new file mode 100644 index 00000000..8a2dfdb5 --- /dev/null +++ b/workflow/lib/idds/workflowv2/work.py @@ -0,0 +1,1932 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 + +import copy +import datetime +import logging +import os +import stat +import uuid +import traceback + +from idds.common import exceptions +from idds.common.constants import (WorkStatus, ProcessingStatus, + CollectionStatus, CollectionType) +from idds.common.utils import setup_logging +from idds.common.utils import str_to_date +# from idds.common.utils import json_dumps + +from .base import Base + + +setup_logging(__name__) + + +class Parameter(object): + def __init__(self, params): + assert(type(params) in [dict]) + self.params = params + + def add(self, name, value): + self.params[name] = value + + def get_param_names(self): + return self.params.keys() + + def get_param_value(self, name): + value = self.params.get(name, None) + if value and callable(value): + value = value() + return value + + +class Collection(Base): + + def __init__(self, scope=None, name=None, coll_type=CollectionType.Dataset, coll_metadata={}): + super(Collection, self).__init__() + self.scope = scope + self.name = name + self.coll_metadata = coll_metadata + + self.collection = None + + self.internal_id = str(uuid.uuid4())[:8] + self.coll_id = None + self.coll_type = coll_type + self.status = CollectionStatus.New + self.substatus = CollectionStatus.New + + @property + def internal_id(self): + return self.get_metadata_item('internal_id') + + @internal_id.setter + def internal_id(self, value): + self.add_metadata_item('internal_id', value) + + @property + def coll_id(self): + return self.get_metadata_item('coll_id', None) + + @coll_id.setter + def coll_id(self, value): + self.add_metadata_item('coll_id', value) + + @property + def status(self): + st = self.get_metadata_item('status', CollectionStatus.New) + if type(st) in [int]: + st = CollectionStatus(st) + return st + + @status.setter + def status(self, value): + self.add_metadata_item('status', value.value if value else value) + if self.collection: + self.collection['status'] = value + + @property + def coll_type(self): + st = self.get_metadata_item('coll_type', CollectionType.Dataset) + if type(st) in [int]: + st = CollectionType(st) + return st + + @coll_type.setter + def coll_type(self, value): + self.add_metadata_item('coll_type', value.value if value else value) + if self.collection: + self.collection['coll_type'] = value + + @property + def substatus(self): + st = self.get_metadata_item('substatus', CollectionStatus.New) + if type(st) in [int]: + st = CollectionStatus(st) + return st + + @substatus.setter + def substatus(self, value): + self.add_metadata_item('substatus', value.value if value else value) + if self.collection: + self.collection['substatus'] = value + + @property + def collection(self): + return self._collection + + @collection.setter + def collection(self, value): + self._collection = value + if self._collection: + self.scope = self._collection['scope'] + self.name = self._collection['name'] + self.coll_metadata = self._collection['coll_metadata'] + self.coll_id = self._collection['coll_id'] + self.coll_type = self._collection['coll_type'] + self.status = self._collection['status'] + self.substatus = self._collection['substatus'] + + +class Processing(Base): + + def __init__(self, processing_metadata={}): + super(Processing, self).__init__() + + self.processing_metadata = processing_metadata + if self.processing_metadata and 'work' in self.processing_metadata: + self.work = self.processing_metadata['work'] + else: + self.work = None + + self.processing = None + + self.internal_id = str(uuid.uuid4())[:8] + self.processing_id = None + self.workload_id = None + self.status = ProcessingStatus.New + self.substatus = ProcessingStatus.New + self.last_updated_at = datetime.datetime.utcnow() + self.polling_retries = 0 + self.tocancel = False + self.tosuspend = False + self.toresume = False + self.toexpire = False + self.tofinish = False + self.toforcefinish = False + self.operation_time = datetime.datetime.utcnow() + self.submitted_at = None + + self.external_id = None + self.errors = None + + self.output_data = None + + self.retries = 0 + + @property + def internal_id(self): + return self.get_metadata_item('internal_id') + + @internal_id.setter + def internal_id(self, value): + self.add_metadata_item('internal_id', value) + + @property + def processing_id(self): + return self.get_metadata_item('processing_id', None) + + @processing_id.setter + def processing_id(self, value): + self.add_metadata_item('processing_id', value) + + @property + def workload_id(self): + return self.get_metadata_item('workload_id', None) + + @workload_id.setter + def workload_id(self, value): + self.add_metadata_item('workload_id', value) + + def get_workload_id(self): + return self.workload_id + + @property + def status(self): + st = self.get_metadata_item('status', ProcessingStatus.New) + if type(st) in [int]: + st = ProcessingStatus(st) + return st + + @status.setter + def status(self, value): + self.add_metadata_item('status', value.value if value else value) + if self.processing: + self.processing['status'] = value + + @property + def substatus(self): + st = self.get_metadata_item('substatus', ProcessingStatus.New) + if type(st) in [int]: + st = ProcessingStatus(st) + return st + + @substatus.setter + def substatus(self, value): + self.add_metadata_item('substatus', value.value if value else value) + if self.processing: + self.processing['substatus'] = value + + @property + def retries(self): + return self.get_metadata_item('retries', 0) + + @retries.setter + def retries(self, value): + self.add_metadata_item('retries', value) + + @property + def last_updated_at(self): + last_updated_at = self.get_metadata_item('last_updated_at', None) + if last_updated_at and type(last_updated_at) in [str]: + last_updated_at = str_to_date(last_updated_at) + return last_updated_at + + @last_updated_at.setter + def last_updated_at(self, value): + self.add_metadata_item('last_updated_at', value) + + @property + def polling_retries(self): + return self.get_metadata_item('polling_retries', 0) + + @polling_retries.setter + def polling_retries(self, value): + self.add_metadata_item('polling_retries', value) + + @property + def tocancel(self): + return self.get_metadata_item('tocancel', False) + + @tocancel.setter + def tocancel(self, value): + old_value = self.get_metadata_item('tocancel', False) + if old_value != value: + self.operation_time = datetime.datetime.utcnow() + self.add_metadata_item('tocancel', value) + + @property + def tosuspend(self): + return self.get_metadata_item('tosuspend', False) + + @tosuspend.setter + def tosuspend(self, value): + old_value = self.get_metadata_item('tosuspend', False) + if old_value != value: + self.operation_time = datetime.datetime.utcnow() + self.add_metadata_item('tosuspend', value) + + @property + def toresume(self): + return self.get_metadata_item('toresume', False) + + @toresume.setter + def toresume(self, value): + old_value = self.get_metadata_item('toresume', False) + if old_value != value: + self.operation_time = datetime.datetime.utcnow() + self.add_metadata_item('toresume', value) + + @property + def toexpire(self): + return self.get_metadata_item('toexpire', False) + + @toexpire.setter + def toexpire(self, value): + old_value = self.get_metadata_item('toexpire', False) + if old_value != value: + self.operation_time = datetime.datetime.utcnow() + self.add_metadata_item('toexpire', value) + + @property + def tofinish(self): + return self.get_metadata_item('tofinish', False) + + @tofinish.setter + def tofinish(self, value): + old_value = self.get_metadata_item('tofinish', False) + if old_value != value: + self.operation_time = datetime.datetime.utcnow() + self.add_metadata_item('tofinish', value) + + @property + def toforcefinish(self): + return self.get_metadata_item('toforcefinish', False) + + @toforcefinish.setter + def toforcefinish(self, value): + old_value = self.get_metadata_item('toforcefinish', False) + if old_value != value: + self.operation_time = datetime.datetime.utcnow() + self.add_metadata_item('toforcefinish', value) + + @property + def operation_time(self): + opt_time = self.get_metadata_item('operation_time', None) + if opt_time and type(opt_time) in [str]: + opt_time = str_to_date(opt_time) + return opt_time + + @operation_time.setter + def operation_time(self, value): + self.add_metadata_item('operation_time', value) + + def in_operation_time(self): + if self.operation_time: + time_diff = datetime.datetime.utcnow() - self.operation_time + time_diff = time_diff.total_seconds() + if time_diff < 120: + return True + return False + + @property + def submitted_at(self): + opt_time = self.get_metadata_item('submitted_at', None) + if opt_time and type(opt_time) in [str]: + opt_time = str_to_date(opt_time) + return opt_time + + @submitted_at.setter + def submitted_at(self, value): + self.add_metadata_item('submitted_at', value) + + @property + def errors(self): + return self.get_metadata_item('errors', None) + + @errors.setter + def errors(self, value): + self.add_metadata_item('errors', value) + + @property + def external_id(self): + return self.get_metadata_item('external_id', None) + + @external_id.setter + def external_id(self, value): + self.add_metadata_item('external_id', value) + + @property + def processing(self): + return self._processing + + @processing.setter + def processing(self, value): + self._processing = value + if self._processing: + self.processing_id = self._processing.get('processing_id', None) + self.workload_id = self._processing.get('workload_id', None) + self.status = self._processing.get('status', None) + self.substatus = self._processing.get('substatus', None) + self.processing_metadata = self._processing.get('processing_metadata', None) + self.submitted_at = self._processing.get('submitted_at', None) + if self.processing_metadata and 'processing' in self.processing_metadata: + proc = self.processing_metadata['processing'] + self.work = proc.work + self.external_id = proc.external_id + self.errors = proc.errors + if not self.submitted_at: + self.submitted_at = proc.submitted_at + + self.output_data = self._processing.get('output_metadata', None) + + def has_new_updates(self): + self.last_updated_at = datetime.datetime.utcnow() + + +class Work(Base): + + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, work_type=None, + work_tag=None, exec_type='local', sandbox=None, work_id=None, work_name=None, + primary_input_collection=None, other_input_collections=None, input_collections=None, + primary_output_collection=None, other_output_collections=None, output_collections=None, + log_collections=None, release_inputs_after_submitting=False, + agent_attributes=None, is_template=False, + logger=None): + """ + Init a work/task/transformation. + + :param setup: A string to setup the executable enviroment, it can be None. + :param executable: The executable. + :param arguments: The arguments. + :param parameters: A dict with arguments needed to be replaced. + :param work_type: The work type like data carousel, hyperparameteroptimization and so on. + :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. + :param sandbox: The sandbox. + :param work_id: The work/task id. + :param primary_input_collection: The primary input collection. + :param other_input_collections: List of the input collections. + :param output_collections: List of the output collections. + # :param workflow: The workflow the current work belongs to. + """ + self._collections = {} + self._primary_input_collection = None + self._primary_output_collection = None + self._other_input_collections = [] + self._other_output_collections = [] + + self._processings = {} + + super(Work, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_work_id = self.internal_id + self.is_template = is_template + self.class_name = self.__class__.__name__.lower() + self.initialized = False + self.sequence_id = 0 + + self.logger = logger + if self.logger is None: + self.setup_logger() + + self.setup = setup + self.executable = executable + self.arguments = arguments + self.parameters = parameters + + self.work_type = work_type + self.work_tag = work_tag + self.exec_type = exec_type + self.sandbox = sandbox + self.work_id = work_id + self.work_name = work_name + if not self.work_name: + self.work_name = self.template_work_id + # self.workflow = workflow + self.transforming = False + self.workdir = None + + self.log_collections = [] + if input_collections and (primary_input_collection or other_input_collections): + raise Exception("input_collections and (primary_input_collection, other_input_collections) cannot be used at the same time.") + if output_collections and (primary_output_collection or other_output_collections): + raise Exception("output_collections and (primary_output_collection, other_output_collections) cannot be used at the same time.") + + if input_collections and type(input_collections) not in [list, tuple]: + input_collections = [input_collections] + if output_collections and type(output_collections) not in [list, tuple]: + output_collections = [output_collections] + + if input_collections: + primary_input_collection = input_collections[0] + if len(input_collections) > 1: + other_input_collections = input_collections[1:] + if output_collections: + primary_output_collection = output_collections[0] + if len(output_collections) > 1: + other_output_collections = output_collections[1:] + + # self.primary_input_collection = primary_input_collection + self.set_primary_input_collection(primary_input_collection) + self.set_primary_output_collection(primary_output_collection) + + # self.other_input_collections = other_input_collections + if other_input_collections and type(other_input_collections) not in [list, tuple]: + other_input_collections = [other_input_collections] + self.add_other_input_collections(other_input_collections) + if other_output_collections and type(other_output_collections) not in [list, tuple]: + other_output_collections = [other_output_collections] + self.add_other_output_collections(other_output_collections) + + # if input_collections and type(input_collections) not in [list, tuple]: + # input_collections = [input_collections] + # self.add_input_collections(input_collections) + # if output_collections and type(output_collections) not in [list, tuple]: + # output_collections = [output_collections] + # self.add_output_collections(output_collections) + + if log_collections and type(log_collections) not in [list, tuple]: + log_collections = [log_collections] + self.add_log_collections(log_collections) + + self.release_inputs_after_submitting = release_inputs_after_submitting + self.has_new_inputs = True + + self.started = False + self.status = WorkStatus.New + self.substatus = WorkStatus.New + self.polling_retries = 0 + self.errors = [] + self.next_works = [] + + self.work_name_to_coll_map = [] + + self.processings = {} + self.active_processings = [] + self.cancelled_processings = [] + self.suspended_processings = [] + self.old_processings = [] + self.terminated_msg = "" + self.output_data = None + self.parameters_for_next_task = None + + self.status_statistics = {} + + self.agent_attributes = agent_attributes + + self.proxy = None + self.original_proxy = None + + self.tocancel = False + self.tosuspend = False + self.toresume = False + self.toexpire = False + self.tofinish = False + self.toforcefinish = False + + self.last_updated_at = datetime.datetime.utcnow() + + self.to_update_processings = {} + + self.backup_to_release_inputs = {'0': [], '1': [], '2': []} + + self.num_run = None + + """ + self._running_data_names = [] + for name in ['internal_id', 'template_work_id', 'initialized', 'sequence_id', 'parameters', 'work_id', 'transforming', 'workdir', + # 'collections', 'primary_input_collection', 'other_input_collections', 'output_collections', 'log_collections', + 'collections', + # 'output_data', + '_has_new_inputs', 'status', 'substatus', 'polling_retries', 'errors', 'next_works', + 'processings', 'active_processings', 'cancelled_processings', 'suspended_processings', 'old_processings', + # 'terminated_msg', 'output_data', 'parameters_for_next_task', 'status_statistics', + 'tocancel', 'tosuspend', 'toresume']: + self._running_data_names.append(name) + """ + + def get_class_name(self): + return self.__class__.__name__ + + def get_internal_id(self): + return self.internal_id + + def get_template_work_id(self): + return self.template_work_id + + def get_sequence_id(self): + return self.sequence_id + + @property + def internal_id(self): + return self.get_metadata_item('internal_id') + + @internal_id.setter + def internal_id(self, value): + self.add_metadata_item('internal_id', value) + + @property + def template_work_id(self): + return self.get_metadata_item('template_work_id') + + @template_work_id.setter + def template_work_id(self, value): + self.add_metadata_item('template_work_id', value) + + @property + def workload_id(self): + return self.get_metadata_item('workload_id', None) + + @workload_id.setter + def workload_id(self, value): + self.add_metadata_item('workload_id', value) + + def get_workload_id(self): + return self.workload_id + + @property + def initialized(self): + return self.get_metadata_item('initialized', False) + + @initialized.setter + def initialized(self, value): + self.add_metadata_item('initialized', value) + + @property + def sequence_id(self): + return self.get_metadata_item('sequence_id', 0) + + @sequence_id.setter + def sequence_id(self, value): + self.add_metadata_item('sequence_id', value) + + @property + def parameters(self): + return self.get_metadata_item('parameters', None) + + @parameters.setter + def parameters(self, value): + self.add_metadata_item('parameters', value) + + @property + def work_id(self): + return self.get_metadata_item('work_id', None) + + @work_id.setter + def work_id(self, value): + self.add_metadata_item('work_id', value) + + @property + def transforming(self): + return self.get_metadata_item('transforming', False) + + @transforming.setter + def transforming(self, value): + self.add_metadata_item('transforming', value) + + @property + def workdir(self): + return self.get_metadata_item('workdir', None) + + @workdir.setter + def workdir(self, value): + self.add_metadata_item('workdir', value) + + @property + def work_name(self): + return self.get_metadata_item('work_name', 0) + + @work_name.setter + def work_name(self, value): + self.add_metadata_item('work_name', value) + + @property + def has_new_inputs(self): + return self.get_metadata_item('has_new_inputs', 0) + + @has_new_inputs.setter + def has_new_inputs(self, value): + self.add_metadata_item('has_new_inputs', value) + + @property + def started(self): + return self.get_metadata_item('started', False) + + @started.setter + def started(self, value): + self.add_metadata_item('started', value) + + @property + def status(self): + st = self.get_metadata_item('status', WorkStatus.New) + if type(st) in [int]: + st = WorkStatus(st) + return st + + @status.setter + def status(self, value): + if not self.transforming: + if value and value in [WorkStatus.Transforming, + WorkStatus.Finished, + WorkStatus.SubFinished, + WorkStatus.Failed, + WorkStatus.Running]: + self.transforming = True + self.add_metadata_item('status', value.value if value else value) + + @property + def substatus(self): + st = self.get_metadata_item('substatus', WorkStatus.New) + if type(st) in [int]: + st = WorkStatus(st) + return st + + @substatus.setter + def substatus(self, value): + self.add_metadata_item('substatus', value.value if value else value) + + @property + def polling_retries(self): + return self.get_metadata_item('polling_retries', 0) + + @polling_retries.setter + def polling_retries(self, value): + self.add_metadata_item('polling_retries', value) + + @property + def errors(self): + return self.get_metadata_item('errors', None) + + @errors.setter + def errors(self, value): + self.add_metadata_item('errors', value) + + @property + def next_works(self): + return self.get_metadata_item('next_works', 0) + + @next_works.setter + def next_works(self, value): + self.add_metadata_item('next_works', value) + + @property + def collections(self): + return self._collections + + @collections.setter + def collections(self, value): + self._collections = value + coll_metadata = {} + if self._collections: + for k in self._collections: + coll = self._collections[k] + if type(coll) in [Collection]: + coll_metadata[k] = {'coll_id': coll.coll_id} + self.add_metadata_item('collections', coll_metadata) + + @property + def processings(self): + return self._processings + + @processings.setter + def processings(self, value): + self._processings = value + proc_metadata = {} + if self._processings: + for k in self._processings: + proc = self._processings[k] + if type(proc) in [Processing]: + proc_metadata[k] = {'processing_id': proc.processing_id} + self.add_metadata_item('processings', proc_metadata) + + def refresh_work(self): + coll_metadata = {} + if self._collections: + for k in self._collections: + coll = self._collections[k] + if type(coll) in [Collection]: + coll_metadata[k] = {'coll_id': coll.coll_id} + self.add_metadata_item('collections', coll_metadata) + + proc_metadata = {} + if self._processings: + for k in self._processings: + proc = self._processings[k] + if type(proc) in [Processing]: + proc_metadata[k] = {'processing_id': proc.processing_id} + self.add_metadata_item('processings', proc_metadata) + + def load_work(self): + coll_metadata = self.get_metadata_item('collections', {}) + for k in self._collections: + if k in coll_metadata: + coll_id = coll_metadata[k]['coll_id'] + self._collections[k].coll_id = coll_id + + proc_metadata = self.get_metadata_item('processings', {}) + for k in self._processings: + if k in proc_metadata: + proc_id = proc_metadata[k]['processing_id'] + self._processings[k].processing_id = proc_id + + def load_metadata(self): + self.load_work() + + @property + def active_processings(self): + return self.get_metadata_item('active_processings', []) + + @active_processings.setter + def active_processings(self, value): + self.add_metadata_item('active_processings', value) + + @property + def cancelled_processings(self): + return self.get_metadata_item('cancelled_processings', []) + + @cancelled_processings.setter + def cancelled_processings(self, value): + self.add_metadata_item('cancelled_processings', value) + + @property + def suspended_processings(self): + return self.get_metadata_item('suspended_processings', []) + + @suspended_processings.setter + def suspended_processings(self, value): + self.add_metadata_item('suspended_processings', value) + + @property + def old_processings(self): + return self.get_metadata_item('old_processings', []) + + @old_processings.setter + def old_processings(self, value): + self.add_metadata_item('old_processings', value) + + @property + def tocancel(self): + return self.get_metadata_item('tocancel', False) + + @tocancel.setter + def tocancel(self, value): + self.add_metadata_item('tocancel', value) + + @property + def tosuspend(self): + return self.get_metadata_item('tosuspend', False) + + @tosuspend.setter + def tosuspend(self, value): + self.add_metadata_item('tosuspend', value) + + @property + def toresume(self): + return self.get_metadata_item('toresume', False) + + @toresume.setter + def toresume(self, value): + self.add_metadata_item('toresume', value) + + @property + def toexpire(self): + return self.get_metadata_item('toexpire', False) + + @toexpire.setter + def toexpire(self, value): + self.add_metadata_item('toexpire', value) + + @property + def tofinish(self): + return self.get_metadata_item('tofinish', False) + + @tofinish.setter + def tofinish(self, value): + self.add_metadata_item('tofinish', value) + + @property + def toforcefinish(self): + return self.get_metadata_item('toforcefinish', False) + + @toforcefinish.setter + def toforcefinish(self, value): + self.add_metadata_item('toforcefinish', value) + + @property + def last_updated_at(self): + last_updated_at = self.get_metadata_item('last_updated_at', None) + if last_updated_at and type(last_updated_at) in [str]: + last_updated_at = str_to_date(last_updated_at) + return last_updated_at + + @last_updated_at.setter + def last_updated_at(self, value): + self.add_metadata_item('last_updated_at', value) + + def has_new_updates(self): + self.last_updated_at = datetime.datetime.utcnow() + + @property + def to_update_processings(self): + return self.get_metadata_item('to_update_processings', {}) + + @to_update_processings.setter + def to_update_processings(self, value): + self.add_metadata_item('to_update_processings', value) + + @property + def num_run(self): + return self.get_metadata_item('num_run', None) + + @num_run.setter + def num_run(self, value): + if value is not None: + self.add_metadata_item('num_run', value) + for k in self._collections: + coll = self._collections[k] + if type(coll) in [Collection]: + coll.name = coll.name + "." + str(value) + + @property + def primary_input_collection(self): + if self._primary_input_collection: + return self.collections[self._primary_input_collection] + return None + + @primary_input_collection.setter + def primary_input_collection(self, value): + self.set_primary_input_collection(value) + + @property + def primary_output_collection(self): + if self._primary_output_collection: + return self.collections[self._primary_output_collection] + return None + + @primary_output_collection.setter + def primary_output_collection(self, value): + self.set_primary_output_collection(value) + + @property + def input_collections(self): + keys = [self._primary_input_collection] + self._other_input_collections + return [self.collections[k] for k in keys] + + @input_collections.setter + def input_collections(self, value): + if value and type(value) not in [list, tuple]: + value = [value] + + if value: + primary_collection = value[0] + other_collections = [] + if len(value) > 1: + other_collections = value[1:] + + self.set_primary_input_collection(primary_collection) + + if other_collections and type(other_collections) not in [list, tuple]: + other_collections = [other_collections] + self.add_other_input_collections(other_collections) + + @property + def output_collections(self): + keys = [self._primary_output_collection] + self._other_output_collections + return [self.collections[k] for k in keys] + + @output_collections.setter + def output_collections(self, value): + if value and type(value) not in [list, tuple]: + value = [value] + + if value: + primary_collection = value[0] + other_collections = [] + if len(value) > 1: + other_collections = value[1:] + + self.set_primary_output_collection(primary_collection) + + if other_collections and type(other_collections) not in [list, tuple]: + other_collections = [other_collections] + self.add_other_output_collections(other_collections) + + def set_work_name(self, work_name): + self.work_name = work_name + + def get_work_name(self): + return self.work_name + + def get_is_template(self): + self.is_template + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) + + def add_errors(self, error): + self.errors.append(error) + + def get_errors(self): + return self.errors + + def set_work_id(self, work_id, transforming=True): + """ + *** Function called by Marshaller agent. + *** It's the transform_id set by core_workprogresses + """ + self.work_id = work_id + self.transforming = transforming + + def get_work_id(self): + """ + *** Function called by Marshaller agent. + """ + return self.work_id + + # def set_workflow(self, workflow): + # self.workflow = workflow + + def clean_work(self): + self.processings = {} + self.active_processings = [] + self.cancelled_processings = [] + self.suspended_processings = [] + self.old_processings = [] + self.terminated_msg = "" + self.output_data = None + self.parameters_for_next_task = None + + def set_agent_attributes(self, attrs, req_attributes=None): + if attrs and self.class_name in attrs: + if self.agent_attributes is None: + self.agent_attributes = {} + for key, value in attrs[self.class_name].items(): + self.agent_attributes[key] = value + self.logger.info("agent_attributes: %s" % self.agent_attributes) + + def get_agent_attributes(self): + return self.agent_attributes + + def set_workdir(self, workdir): + self.workdir = workdir + + def get_workdir(self): + return self.workdir + + def set_status(self, status): + """ + *** Function called by Marshaller agent. + """ + assert(isinstance(status, WorkStatus)) + self.status = status + # if self.workflow: + # self.workflow.work_status_update_trigger(self, status) + + def get_status(self): + return self.status + + def set_terminated_msg(self, msg): + """ + *** Function called by Marshaller agent. + """ + self.terminated_msg = msg + self.add_errors(msg) + + def get_terminated_msg(self): + return self.terminated_msg + + def set_output_data(self, data): + self.output_data = data + + def get_output_data(self): + return self.output_data + + def set_parameters_for_next_task(self, params): + self.parameters_for_next_task = params + + def get_parameters_for_next_task(self): + return self.parameters_for_next_task + + def __eq__(self, obj): + if self.work_id == obj.work_id: + return True + return False + + def __hash__(self): + return self.work_id + + def __str__(self): + return str(self.to_dict()) + + def get_work_type(self): + """ + *** Function called by Marshaller agent. + """ + return self.work_type + + def get_work_tag(self): + """ + *** Function called by Marshaller agent. + """ + return self.work_tag + + def set_parameters(self, parameters): + self.parameters = parameters + for p in self.parameters: + if self.parameters[p] is not None and hasattr(self, p): + fp = getattr(self, p) + fp = self.parameters[p] # noqa F841 + + def get_parameters(self): + return self.parameters + + def set_arguments(self, arguments): + self.arguments = arguments + + def get_arguments(self): + return self.arguments + + def has_to_release_inputs(self): + if self.backup_to_release_inputs['0'] or self.backup_to_release_inputs['1'] or self.backup_to_release_inputs['2']: + return True + return False + + def add_backup_to_release_inputs(self, to_release_inputs): + if to_release_inputs: + self.backup_to_release_inputs['0'] = self.backup_to_release_inputs['0'] + to_release_inputs + + def get_backup_to_release_inputs(self): + to_release_inputs = self.backup_to_release_inputs['0'] + self.backup_to_release_inputs['1'] + self.backup_to_release_inputs['2'] + self.backup_to_release_inputs['2'] = self.backup_to_release_inputs['1'] + self.backup_to_release_inputs['1'] = self.backup_to_release_inputs['0'] + self.backup_to_release_inputs['0'] = [] + return to_release_inputs + + def is_started(self): + return self.started + + def is_running(self): + if self.status in [WorkStatus.Running]: + return True + return False + + def is_terminated(self): + """ + *** Function called by Transformer agent. + """ + if (self.status in [WorkStatus.Finished, WorkStatus.SubFinished, WorkStatus.Failed, WorkStatus.Cancelled, WorkStatus.Suspended, WorkStatus.Expired] + and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]): # noqa W503 + return True + return False + + def is_finished(self): + """ + *** Function called by Transformer agent. + """ + if self.status in [WorkStatus.Finished] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]: + return True + return False + + def is_subfinished(self): + """ + *** Function called by Transformer agent. + """ + if self.status in [WorkStatus.SubFinished] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]: + return True + return False + + def is_failed(self): + """ + *** Function called by Transformer agent. + """ + if self.status in [WorkStatus.Failed] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]: + return True + return False + + def is_expired(self): + """ + *** Function called by Transformer agent. + """ + if self.status in [WorkStatus.Expired] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]: + return True + return False + + def is_cancelled(self): + """ + *** Function called by Transformer agent. + """ + if self.status in [WorkStatus.Cancelled] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]: + return True + return False + + def is_suspended(self): + """ + *** Function called by Transformer agent. + """ + if self.status in [WorkStatus.Suspended] and self.substatus not in [WorkStatus.ToCancel, WorkStatus.ToSuspend, WorkStatus.ToResume]: + return True + return False + + def add_next_work(self, work): + self.next_works.append(work) + + def parse_arguments(self): + try: + arguments = self.get_arguments() + parameters = self.get_parameters() + arguments = arguments.format(**parameters) + return arguments + except Exception as ex: + self.add_errors(str(ex)) + + def set_initialized(self): + self.initialized = True + + def unset_initialized(self): + self.initialized = False + + def is_initialized(self): + return self.initialized + + def initialize_work(self): + if self.parameters and self.arguments: + # for key in self.parameters.get_param_names(): + # self.arguments = re.sub(key, str(self.parameters.get_param_value(key)), self.arguments) + # self.arguments = self.arguments.format(**self.parameters) + pass + if not self.is_initialized(): + self.set_initialized() + + def copy(self): + new_work = copy.deepcopy(self) + return new_work + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + def depend_on(self, work): + return False + + def generate_work_from_template(self): + logger = self.logger + self.logger = None + new_work = copy.deepcopy(self) + self.logger = logger + new_work.logger = logger + # new_work.template_work_id = self.get_internal_id() + if self.is_template: + new_work.internal_id = str(uuid.uuid4())[:8] + return new_work + + def get_template_id(self): + return self.template_work_id + + def resume_work(self): + if self.status in [WorkStatus.New, WorkStatus.Ready]: + pass + else: + self.status = WorkStatus.Transforming + self.polling_retries = 0 + + def add_collection_to_collections(self, coll): + assert(isinstance(coll, dict)) + assert('scope' in coll) + assert('name' in coll) + + coll_metadata = copy.copy(coll) + del coll_metadata['scope'] + del coll_metadata['name'] + if 'type' in coll_metadata: + coll_type = coll_metadata['type'] + del coll_metadata['type'] + else: + coll_type = CollectionType.Dataset + + collection = Collection(scope=coll['scope'], name=coll['name'], coll_type=coll_type, coll_metadata=coll_metadata) + self.collections[collection.internal_id] = collection + return collection + + def set_primary_input_collection(self, coll): + if coll: + collection = self.add_collection_to_collections(coll) + self._primary_input_collection = collection.internal_id + + def get_primary_input_collection(self): + """ + *** Function called by Marshaller agent. + """ + if self._primary_input_collection: + return self.collections[self._primary_input_collection] + return None + + def set_primary_output_collection(self, coll): + if coll: + collection = self.add_collection_to_collections(coll) + self._primary_output_collection = collection.internal_id + + def get_primary_output_collection(self): + """ + *** Function called by Marshaller agent. + """ + if self._primary_output_collection: + return self.collections[self._primary_output_collection] + return None + + def add_other_input_collections(self, colls): + if not colls: + return + if type(colls) not in [list, tuple]: + colls = [colls] + + for coll in colls: + collection = self.add_collection_to_collections(coll) + self._other_input_collections.append(collection.internal_id) + + def get_other_input_collections(self): + return [self.collections[k] for k in self._other_input_collections] + + def add_other_output_collections(self, colls): + if not colls: + return + if type(colls) not in [list, tuple]: + colls = [colls] + + for coll in colls: + collection = self.add_collection_to_collections(coll) + self._other_output_collections.append(collection.internal_id) + + def get_other_output_collections(self): + return [self.collections[k] for k in self._other_output_collections] + + def get_input_collections(self): + """ + *** Function called by Transformer agent. + """ + keys = [self._primary_input_collection] + self.other_input_collections + return [self.collections[k] for k in keys] + + def get_output_collections(self): + """ + *** Function called by Transformer agent. + """ + keys = [self._primary_output_collection] + self.other_output_collections + return [self.collections[k] for k in keys] + + def is_input_collections_closed(self): + colls = self.get_input_collections() + for coll in colls: + if coll.status not in [CollectionStatus.Closed]: + return False + return True + + def is_internal_collection(self, coll): + if (coll.coll_metadata and 'source' in coll.coll_metadata and coll.coll_metadata['source'] # noqa W503 + and type(coll.coll_metadata['source']) == str and coll.coll_metadata['source'].lower() == 'idds'): # noqa W503 + return True + return False + + def get_internal_collections(self, coll): + if coll.coll_metadata and 'request_id' in coll.coll_metadata: + # relation_type = coll['coll_metadata']['relation_type'] if 'relation_type' in coll['coll_metadata'] else CollectionRelationType.Output + # colls = core_catalog.get_collections(scope=coll['scope'], + # name=coll['name'], + # request_id=coll['coll_metadata']['request_id'], + # relation_type=relation_type) + return [] + return [] + + def poll_internal_collection(self, coll): + try: + if coll.status in [CollectionStatus.Closed]: + return coll + else: + if coll.coll_metadata is None: + coll.coll_metadata = {} + coll.coll_metadata['bytes'] = 0 + coll.coll_metadata['availability'] = 0 + coll.coll_metadata['events'] = 0 + coll.coll_metadata['is_open'] = True + coll.coll_metadata['run_number'] = 1 + coll.coll_metadata['did_type'] = 'DATASET' + coll.coll_metadata['list_all_files'] = False + coll.coll_metadata['interal_colls'] = [] + + is_open = False + internal_colls = self.get_internal_collections(coll) + for i_coll in internal_colls: + if i_coll.status not in [CollectionStatus.Closed]: + is_open = True + coll.coll_metadata['bytes'] += i_coll['bytes'] + + if not is_open: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + if len(internal_colls) > 1: + coll.coll_metadata['coll_type'] = CollectionType.Container + else: + coll.coll_metadata['coll_type'] = CollectionType.Dataset + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_internal_input_contents(self, coll): + """ + Get all input contents from iDDS collections. + """ + coll = self.collections[self.primary_input_collection] + internal_colls = self.get_internal_collection(coll) + internal_coll_ids = [coll.coll_id for coll in internal_colls] + if internal_coll_ids: + # contents = catalog.get_contents_by_coll_id_status(coll_id=coll_ids) + contents = [] + else: + contents = [] + return contents + + def get_input_contents(self): + """ + Get all input contents from DDM. + """ + pass + + def add_output_collections(self, colls): + """ + """ + if not colls: + return + if type(colls) not in [list, tuple]: + colls = [colls] + + value = colls + if value: + primary_collection = value[0] + other_collections = [] + if len(value) > 1: + other_collections = value[1:] + + self.set_primary_output_collection(primary_collection) + + if other_collections and type(other_collections) not in [list, tuple]: + other_collections = [other_collections] + self.add_other_output_collections(other_collections) + + def get_output_contents(self): + pass + + def add_log_collections(self, colls): + if not colls: + return + if type(colls) not in [list, tuple]: + colls = [colls] + + for coll in colls: + collection = self.add_collection_to_collections(coll) + self.log_collections.append(collection.internal_id) + + def get_log_collections(self): + return [self.collections[k] for k in self.log_collections] + + def set_has_new_inputs(self, yes=True): + self.has_new_inputs = yes + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + *** Function called by Transformer agent. + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + inputs = self.get_input_contents() + # mapped_inputs = mapped_input_output_maps.keys() + + mapped_inputs = [] + for map_id in mapped_input_output_maps: + map_id_inputs = mapped_input_output_maps[map_id] + for ip in map_id_inputs: + if ip['coll_id'] == self.primary_input_collection['coll_id']: + mapped_inputs.append(ip['scope'] + ':' + ip['name']) + + new_inputs = [] + for ip in inputs: + if ip in mapped_inputs: + pass + else: + new_inputs.append(ip) + + new_input_output_maps = {} + mapped_keys = mapped_input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + for ip in new_inputs: + self.num_mapped_inputs += 1 + out_ip = copy.deepcopy(ip) + out_ip['coll_id'] = self.collections[self.output_collections[0]]['coll_id'] + new_input_output_maps[next_key] = {'inputs': [ip], + 'outputs': [out_ip], + 'inputs_dependency': [], + 'logs': []} + next_key += 1 + + return new_input_output_maps + + def set_collection_id(self, collection, coll_id): + # print(collection) + # print(coll_id) + self.collections[collection.internal_id]['coll_id'] = coll_id + + def should_release_inputs(self, processing=None, poll_operation_time_period=120): + if self.release_inputs_after_submitting: + if not poll_operation_time_period: + poll_operation_time_period = 120 + + processing_model = processing.processing + if (processing_model and processing_model['submitted_at'] # noqa: W503 + and processing_model['submitted_at'] + datetime.timedelta(seconds=int(poll_operation_time_period)) # noqa: W503 + < datetime.datetime.utcnow()): # noqa: W503 + + if (processing and processing.status + and processing.status not in [ProcessingStatus.New, ProcessingStatus.New.value]): # noqa: W503 + # and processing.status not in [ProcessingStatus.New, ProcessingStatus.New.value, # noqa: W503 + # ProcessingStatus.Submitting, ProcessingStatus.Submitting.value]): # noqa: W503 + return True + return False + return True + + def use_dependency_to_release_jobs(self): + """ + *** Function called by Transformer agent. + """ + return False + + def set_work_name_to_coll_map(self, work_name_to_coll_map): + self.work_name_to_coll_map = work_name_to_coll_map + + def get_work_name_to_coll_map(self): + return self.work_name_to_coll_map + + def add_processing_to_processings(self, processing): + # assert(isinstance(processing, dict)) + # assert('processing_metadata' in processing) + # if 'processing_metadata' not in processing: + # processing['processing_metadata'] = {} + + # if 'internal_id' not in processing['processing_metadata']: + # processing['processing_metadata']['internal_id'] = str(uuid.uuid1()) + self.processings[processing.internal_id] = processing + + def get_processing_ids(self): + ids = [] + for p_id in self.active_processings: + p = self.processings[p_id] + if p.processing_id: + ids.append(p.processing_id) + return ids + + # def set_processing(self, processing): + # self.processing = processing + + def set_processing_id(self, processing, processing_id): + """ + *** Function called by Transformer agent. + """ + self.processings[processing.internal_id].processing_id = processing_id + + def set_processing_status(self, processing, status, substatus): + """ + *** Function called by Transformer agent. + """ + self.processings[processing.internal_id].status = status + self.processings[processing.internal_id].substatus = substatus + # if status not in [ProcessingStatus.New, ProcessingStatus.Submitting, + # ProcessingStatus.Submitted, ProcessingStatus.Running]: + # if processing['processing_metadata']['internal_id'] in self.active_processings: + # del self.active_processings[processing['processing_metadata']['internal_id']] + + def set_processing_output_metadata(self, processing, output_metadata): + """ + *** Function called by Transformer agent. + """ + processing = self.processings[processing.internal_id] + # processing['output_metadata'] = output_metadata + self.set_output_data(output_metadata) + + def is_processing_substatus_new_operationing(self, processing): + if processing.substatus in [ProcessingStatus.ToCancel, + ProcessingStatus.ToSuspend, + ProcessingStatus.ToResume, + ProcessingStatus.ToFinish, + ProcessingStatus.ToForceFinish]: + return True + return False + + def is_processing_terminated(self, processing): + self.logger.debug("is_processing_terminated: status: %s, substatus: %s" % (processing.status, processing.substatus)) + if self.is_processing_substatus_new_operationing(processing): + return False + + if processing.status not in [ProcessingStatus.New, + ProcessingStatus.Submitting, + ProcessingStatus.Submitted, + ProcessingStatus.Running, + ProcessingStatus.ToCancel, + ProcessingStatus.Cancelling, + ProcessingStatus.ToSuspend, + ProcessingStatus.Suspending, + ProcessingStatus.ToResume, + ProcessingStatus.Resuming, + ProcessingStatus.ToFinish, + ProcessingStatus.ToForceFinish]: + return True + return False + + def reap_processing(self, processing): + if self.is_processing_terminated(processing): + self.active_processings.remove(processing.internal_id) + else: + self.logger.error("Cannot reap an unterminated processing: %s" % processing) + + def is_processings_started(self): + """ + *** Function called by Transformer agent. + """ + # for p_id in self.active_processings: + for p_id in self.processings: + p = self.processings[p_id] + if p.submitted_at: + return True + return False + + def is_processings_running(self): + """ + *** Function called by Transformer agent. + """ + for p_id in self.active_processings: + p = self.processings[p_id] + if p.status in [ProcessingStatus.Running]: + return True + return False + + def is_processings_terminated(self): + """ + *** Function called by Transformer agent. + """ + for p_id in self.active_processings: + p = self.processings[p_id] + if self.is_processing_terminated(p): + pass + else: + return False + return True + + def is_processings_finished(self): + """ + *** Function called by Transformer agent. + """ + for p_id in self.active_processings: + p = self.processings[p_id] + if not self.is_processing_terminated(p) or p.status not in [ProcessingStatus.Finished]: + return False + return True + + def is_processings_subfinished(self): + """ + *** Function called by Transformer agent. + """ + has_finished = False + has_failed = False + for p_id in self.active_processings: + p = self.processings[p_id] + if not self.is_processing_terminated(p): + return False + else: + if p.status in [ProcessingStatus.Finished]: + has_finished = True + if p.status in [ProcessingStatus.Failed]: + has_failed = True + if has_finished and has_failed: + return True + return False + + def is_processings_failed(self): + """ + *** Function called by Transformer agent. + """ + for p_id in self.active_processings: + p = self.processings[p_id] + if not self.is_processing_terminated(p) or p.status not in [ProcessingStatus.Failed]: + return False + return True + + def is_processings_expired(self): + """ + *** Function called by Transformer agent. + """ + has_expired = False + for p_id in self.active_processings: + p = self.processings[p_id] + if not self.is_processing_terminated(p): + return False + elif p.status in [ProcessingStatus.Expired]: + has_expired = True + if has_expired: + return True + return False + + def is_processings_cancelled(self): + """ + *** Function called by Transformer agent. + """ + has_cancelled = False + for p_id in self.active_processings: + p = self.processings[p_id] + if not self.is_processing_terminated(p): + return False + elif p.status in [ProcessingStatus.Cancelled]: + has_cancelled = True + if has_cancelled: + return True + return False + + def is_processings_suspended(self): + """ + *** Function called by Transformer agent. + """ + has_suspended = False + for p_id in self.active_processings: + p = self.processings[p_id] + if not self.is_processing_terminated(p): + return False + elif p.status in [ProcessingStatus.Suspended]: + has_suspended = True + if has_suspended: + return True + return False + + def create_processing(self, input_output_maps=[]): + """ + *** Function called by Transformer agent. + """ + # proc = {'processing_metadata': {'internal_id': str(uuid.uuid1())}} + proc = Processing() + self.add_processing_to_processings(proc) + self.active_processings.append(proc.internal_id) + return proc + + def get_processing(self, input_output_maps, without_creating=False): + """ + *** Function called by Transformer agent. + """ + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + if not without_creating: + # return None + return self.create_processing(input_output_maps) + # self.process = process + # return process + return None + + def sync_processing(self, processing, processing_model): + processing.processing = processing_model + self.set_processing_status(processing, processing_model['status'], processing_model['substatus']) + + def submit_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + raise exceptions.NotImplementedException + + def abort_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + # raise exceptions.NotImplementedException + self.tocancel = True + if (processing and 'processing_metadata' in processing and processing['processing_metadata'] # noqa W503 + and 'processing' in processing['processing_metadata'] and processing['processing_metadata']['processing']): # noqa W503 + proc = processing['processing_metadata']['processing'] + proc.tocancel = True + + def suspend_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + # raise exceptions.NotImplementedException + self.tosuspend = True + if (processing and 'processing_metadata' in processing and processing['processing_metadata'] # noqa W503 + and 'processing' in processing['processing_metadata'] and processing['processing_metadata']['processing']): # noqa W503 + proc = processing['processing_metadata']['processing'] + proc.tosuspend = True + + def resume_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + # raise exceptions.NotImplementedException + self.toresume = True + if (processing and 'processing_metadata' in processing and processing['processing_metadata'] # noqa W503 + and 'processing' in processing['processing_metadata'] and processing['processing_metadata']['processing']): # noqa W503 + proc = processing['processing_metadata']['processing'] + proc.toresume = True + + def expire_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + # raise exceptions.NotImplementedException + self.toexpire = True + if (processing and 'processing_metadata' in processing and processing['processing_metadata'] # noqa W503 + and 'processing' in processing['processing_metadata'] and processing['processing_metadata']['processing']): # noqa W503 + proc = processing['processing_metadata']['processing'] + proc.toexpire = True + + def finish_processing(self, processing, forcing=False): + """ + *** Function called by Carrier agent. + """ + # raise exceptions.NotImplementedException + if forcing: + self.toforcefinish = True + else: + self.tofinish = True + if (processing and 'processing_metadata' in processing and processing['processing_metadata'] # noqa W503 + and 'processing' in processing['processing_metadata'] and processing['processing_metadata']['processing']): # noqa W503 + proc = processing['processing_metadata']['processing'] + proc.tofinish = True + if forcing: + proc.toforcefinish = True + + def poll_processing_updates(self, processing, input_output_maps): + """ + *** Function called by Carrier agent. + """ + processing['processing'].has_new_updates() + raise exceptions.NotImplementedException + + def is_all_outputs_flushed(self, input_output_maps): + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + + for content in outputs: + if content['status'] != content['substatus']: + return False + return True + + def syn_work_status(self, input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + """ + *** Function called by Transformer agent. + """ + # raise exceptions.NotImplementedException + self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) + self.logger.debug("syn_work_status(%s): is_input_collections_closed: %s" % (str(self.get_processing_ids()), str(self.is_input_collections_closed()))) + self.logger.debug("syn_work_status(%s): has_new_inputs: %s" % (str(self.get_processing_ids()), str(self.has_new_inputs))) + self.logger.debug("syn_work_status(%s): has_to_release_inputs: %s" % (str(self.get_processing_ids()), str(self.has_to_release_inputs()))) + self.logger.debug("syn_work_status(%s): to_release_input_contents: %s" % (str(self.get_processing_ids()), str(to_release_input_contents))) + if self.is_processings_terminated() and self.is_input_collections_closed() and not self.has_new_inputs and not self.has_to_release_inputs() and not to_release_input_contents: + if not self.is_all_outputs_flushed(input_output_maps): + self.logger.warn("The work processings %s is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform" % str(self.get_processing_ids())) + return + + if self.is_processings_finished(): + self.status = WorkStatus.Finished + elif self.is_processings_subfinished(): + self.status = WorkStatus.SubFinished + elif self.is_processings_failed(): + self.status = WorkStatus.Failed + elif self.is_processings_expired(): + self.status = WorkStatus.Expired + elif self.is_processings_cancelled(): + self.status = WorkStatus.Cancelled + elif self.is_processings_suspended(): + self.status = WorkStatus.Suspended + elif self.is_processings_running(): + self.status = WorkStatus.Running + else: + self.status = WorkStatus.Transforming + + if self.is_processings_terminated() or self.is_processings_running() or self.is_processings_started(): + self.started = True + self.logger.debug("syn_work_status(%s): work.status: %s" % (str(self.get_processing_ids()), str(self.status))) + + def sync_work_data(self, status, substatus, work): + # self.status = work.status + work.work_id = self.work_id + work.transforming = self.transforming + self.metadata = work.metadata + + self.status_statistics = work.status_statistics + self.processings = work.processings + + """ + self.status = WorkStatus(status.value) + self.substatus = WorkStatus(substatus.value) + self.workdir = work.workdir + self.has_new_inputs = work.has_new_inputs + self.errors = work.errors + self.next_works = work.next_works + + self.terminated_msg = work.terminated_msg + self.output_data = work.output_data + self.parameters_for_next_task = work.parameters_for_next_task + + self.status_statistics = work.status_statistics + + self.processings = work.processings + self.active_processings = work.active_processings + self.cancelled_processings = work.cancelled_processings + self.suspended_processings = work.suspended_processings + """ + + def add_proxy(self, proxy): + self.proxy = proxy + + def get_proxy(self): + return self.proxy + + def set_user_proxy(self): + if 'X509_USER_PROXY' in os.environ: + self.original_proxy = os.environ['X509_USER_PROXY'] + if self.get_proxy(): + user_proxy = '/tmp/idds_user_proxy' + with open(user_proxy, 'w') as fp: + fp.write(self.get_proxy()) + os.chmod(user_proxy, stat.S_IRUSR | stat.S_IWUSR) + os.environ['X509_USER_PROXY'] = user_proxy + + def unset_user_proxy(self): + if self.original_proxy: + os.environ['X509_USER_PROXY'] = self.original_proxy + else: + del os.environ['X509_USER_PROXY'] diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py new file mode 100644 index 00000000..26bb72d5 --- /dev/null +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -0,0 +1,1825 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + +import copy +import datetime +import logging +import inspect +import random +import time +import uuid + + +from idds.common import exceptions +from idds.common.constants import IDDSEnum, WorkStatus +from idds.common.utils import json_dumps, setup_logging, get_proxy +from idds.common.utils import str_to_date +from .base import Base +from .work import Work + + +setup_logging(__name__) + + +class ConditionOperator(IDDSEnum): + And = 0 + Or = 1 + + +class ConditionTrigger(IDDSEnum): + NotTriggered = 0 + ToTrigger = 1 + Triggered = 2 + + +class CompositeCondition(Base): + def __init__(self, operator=ConditionOperator.And, conditions=[], true_works=None, false_works=None, logger=None): + self._conditions = [] + self._true_works = [] + self._false_works = [] + + super(CompositeCondition, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_id = self.internal_id + # self.template_id = str(uuid.uuid4())[:8] + + self.logger = logger + if self.logger is None: + self.setup_logger() + + if conditions is None: + conditions = [] + if true_works is None: + true_works = [] + if false_works is None: + false_works = [] + if conditions and type(conditions) not in [tuple, list]: + conditions = [conditions] + if true_works and type(true_works) not in [tuple, list]: + true_works = [true_works] + if false_works and type(false_works) not in [tuple, list]: + false_works = [false_works] + self.validate_conditions(conditions) + + self.operator = operator + self.conditions = [] + self.true_works = [] + self.false_works = [] + + self.conditions = conditions + self.true_works = true_works + self.false_works = false_works + + def get_class_name(self): + return self.__class__.__name__ + + def get_internal_id(self): + return self.internal_id + + def get_template_id(self): + return self.template_id + + def copy(self): + new_cond = copy.deepcopy(self) + return new_cond + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + @property + def conditions(self): + # return self.get_metadata_item('true_works', []) + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + + @property + def true_works(self): + # return self.get_metadata_item('true_works', []) + return self._true_works + + @true_works.setter + def true_works(self, value): + self._true_works = value + true_work_meta = self.get_metadata_item('true_works', {}) + for work in value: + if work is None: + continue + if isinstance(work, Work): + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, CompositeCondition): + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, Workflow): + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + self.add_metadata_item('true_works', true_work_meta) + + @property + def false_works(self): + # return self.get_metadata_item('false_works', []) + return self._false_works + + @false_works.setter + def false_works(self, value): + self._false_works = value + false_work_meta = self.get_metadata_item('false_works', {}) + for work in value: + if work is None: + continue + if isinstance(work, Work): + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, CompositeCondition): + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + elif isinstance(work, Workflow): + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + self.add_metadata_item('false_works', false_work_meta) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + for cond in conditions: + assert(inspect.ismethod(cond)) + + def add_condition(self, cond): + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + + # self.conditions.append({'condition': cond, 'current_work': cond.__self__}) + + self._conditions.append(cond) + + def load_metadata(self): + # conditions = self.get_metadata_item('conditions', []) + # true_works_meta = self.get_metadata_item('true_works', {}) + # false_works_meta = self.get_metadata_item('false_works', {}) + pass + + def to_dict(self): + # print('to_dict') + ret = {'class': self.__class__.__name__, + 'module': self.__class__.__module__, + 'attributes': {}} + for key, value in self.__dict__.items(): + # print(key) + # print(value) + # if not key.startswith('__') and not key.startswith('_'): + if not key.startswith('__'): + if key == 'logger': + value = None + elif key == '_conditions': + new_value = [] + for cond in value: + if inspect.ismethod(cond): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id()} + else: + new_cond = cond + new_value.append(new_cond) + value = new_value + elif key in ['_true_works', '_false_works']: + new_value = [] + for w in value: + if isinstance(w, Work): + new_w = w.get_internal_id() + elif isinstance(w, CompositeCondition): + new_w = w.to_dict() + elif isinstance(w, Workflow): + new_w = w.to_dict() + else: + new_w = w + new_value.append(new_w) + value = new_value + else: + value = self.to_dict_l(value) + ret['attributes'][key] = value + return ret + + def get_work_from_id(self, work_id, works): + return works[work_id] + + def load_conditions(self, works): + new_conditions = [] + for cond in self.conditions: + if callable(cond): + new_conditions.append(cond) + else: + if 'idds_method' in cond and 'idds_method_internal_id' in cond: + internal_id = cond['idds_method_internal_id'] + work = self.get_work_from_id(internal_id, works) + if work is not None: + new_cond = getattr(work, cond['idds_method']) + else: + self.logger.error("Work cannot be found for %s" % (internal_id)) + new_cond = cond + else: + new_cond = cond + new_conditions.append(new_cond) + self.conditions = new_conditions + + new_true_works = [] + for w in self.true_works: + if isinstance(w, CompositeCondition) or isinstance(w, Workflow): + # work = w.load_conditions(works, works_template) + w.load_conditions(works) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, works) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_true_works.append(work) + self.true_works = new_true_works + + new_false_works = [] + for w in self.false_works: + if isinstance(w, CompositeCondition) or isinstance(w, Workflow): + # work = w.load_condtions(works, works_template) + w.load_conditions(works) + work = w + elif type(w) in [str]: + work = self.get_work_from_id(w, works) + if work is None: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + else: + self.logger.error("Work cannot be found for %s" % str(w)) + work = w + new_false_works.append(work) + self.false_works = new_false_works + + def all_works(self): + works = [] + works = works + self.all_pre_works() + works = works + self.all_next_works() + return works + + def all_condition_ids(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__.get_internal_id()) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_condition_ids() + return works + + def all_pre_works(self): + works = [] + for cond in self.conditions: + if inspect.ismethod(cond): + works.append(cond.__self__) + else: + self.logger.error("cond cannot be recognized: %s" % str(cond)) + works.append(cond) + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_pre_works() + return works + + def all_next_works(self): + works = [] + for work in self.true_works + self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.all_next_works() + else: + works.append(work) + return works + + def get_current_cond_status(self, cond): + if callable(cond): + if cond(): + return True + else: + return False + else: + if cond: + return True + else: + return False + + def get_cond_status(self): + if self.operator == ConditionOperator.And: + for cond in self.conditions: + if not self.get_current_cond_status(cond): + return False + return True + else: + for cond in self.conditions: + if self.get_current_cond_status(cond): + return True + return False + + def get_condition_status(self): + return self.get_cond_status() + + def is_condition_true(self): + if self.get_cond_status(): + return True + return False + + def is_condition_false(self): + if not self.get_cond_status(): + return True + return False + + def get_next_works(self, trigger=ConditionTrigger.NotTriggered): + works = [] + if self.get_cond_status(): + true_work_meta = self.get_metadata_item('true_works', {}) + for work in self.true_works: + if isinstance(work, CompositeCondition): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_internal_id() not in true_work_meta: + true_work_meta[work.get_internal_id()] = {'triggered': False} + if trigger == ConditionTrigger.ToTrigger: + if not true_work_meta[work.get_internal_id()]['triggered']: + true_work_meta[work.get_internal_id()]['triggered'] = True + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not true_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if true_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + self.add_metadata_item('true_works', true_work_meta) + else: + false_work_meta = self.get_metadata_item('false_works', {}) + for work in self.false_works: + if isinstance(work, CompositeCondition): + works = works + work.get_next_works(trigger=trigger) + else: + if work.get_internal_id() not in false_work_meta: + false_work_meta[work.get_internal_id()] = {'triggered': False} + if trigger == ConditionTrigger.ToTrigger: + if not false_work_meta[work.get_internal_id()]['triggered']: + false_work_meta[work.get_internal_id()]['triggered'] = True + works.append(work) + elif trigger == ConditionTrigger.NotTriggered: + if not false_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + elif trigger == ConditionTrigger.Triggered: + if false_work_meta[work.get_internal_id()]['triggered']: + works.append(work) + self.add_metadata_item('false_works', false_work_meta) + return works + + +class AndCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(AndCondition, self).__init__(operator=ConditionOperator.And, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class OrCondition(CompositeCondition): + def __init__(self, conditions=[], true_works=None, false_works=None, logger=None): + super(OrCondition, self).__init__(operator=ConditionOperator.Or, + conditions=conditions, + true_works=true_works, + false_works=false_works, + logger=logger) + + +class Condition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + super(Condition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + # to support load from old conditions + @property + def cond(self): + # return self.get_metadata_item('true_works', []) + return self.conditions[0] if len(self.conditions) >= 1 else None + + @cond.setter + def cond(self, value): + self.conditions = [value] + + @property + def true_work(self): + # return self.get_metadata_item('true_works', []) + return self.true_works if len(self.true_works) >= 1 else None + + @true_work.setter + def true_work(self, value): + self.true_works = [value] + + @property + def false_work(self): + # return self.get_metadata_item('true_works', []) + return self.false_works if len(self.false_works) >= 1 else None + + @false_work.setter + def false_work(self, value): + self.false_works = [value] + + +class TemplateCondition(CompositeCondition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None, logger=None): + if true_work is not None and not isinstance(true_work, Work): + raise exceptions.IDDSException("true_work can only be set with Work class") + if false_work is not None and not isinstance(false_work, Work): + raise exceptions.IDDSException("false_work can only be set with Work class") + + super(TemplateCondition, self).__init__(operator=ConditionOperator.And, + conditions=[cond] if cond else [], + true_works=[true_work] if true_work else [], + false_works=[false_work] if false_work else [], + logger=logger) + + def validate_conditions(self, conditions): + if type(conditions) not in [tuple, list]: + raise exceptions.IDDSException("conditions must be list") + if len(conditions) > 1: + raise exceptions.IDDSException("Condition class can only support one condition. To support multiple condition, please use CompositeCondition.") + for cond in conditions: + assert(inspect.ismethod(cond)) + assert(isinstance(cond.__self__, Work)) + + def add_condition(self, cond): + raise exceptions.IDDSException("Condition class doesn't support add_condition. To support multiple condition, please use CompositeCondition.") + + +class ParameterLink(Base): + def __init__(self, parameters): + self.parameters = {} + self.num_parameters = 0 + if parameters: + if type(parameters) not in [list, tuple]: + parameters = [parameters] + for p in parameters: + if p: + if type(p) in [str]: + self.parameters[str(self.num_parameters)] = {'source': p, 'destination': p} + self.num_parameters += 1 + elif type(p) in [dict] and 'source' in p and 'destination' in p: + self.parameters[str(self.num_parameters)] = {'source': p['source'], 'destination': p['destination']} + self.num_parameters += 1 + else: + raise Exception("Cannot parse the parameters format. Accepted format: list of string or dict{'source': <>, 'destination': <>}") + + self.internal_id = str(uuid.uuid4())[:8] + self.template_id = self.internal_id + + def get_internal_id(self): + return self.internal_id + + def get_parameter_value(self, work, p): + p_f = getattr(work, p, 'None') + if p_f: + if callable(p_f): + return p_f() + else: + return p_f + else: + return None + + def set_parameters(self, work): + p_values = [] + for p in self.parameters: + p_values[p] = self.get_parameter_value(work, self.parameters[p]['source']) + self.add_metadata_item('parameters', p_values) + + def get_parameters(self): + p_values = self.get_metadata_item('parameters', {}) + ret = {} + for p in self.parameters: + if p in p_values: + ret[self.parameters[p]['destination']] = p_values[p] + + +class WorkflowBase(Base): + + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + """ + Init a workflow. + """ + self._works = {} + self._conditions = {} + self._work_conds = {} + + self.parameter_links = {} + self.parameter_links_source = {} + self.parameter_links_destination = {} + + super(WorkflowBase, self).__init__() + + self.internal_id = str(uuid.uuid4())[:8] + self.template_work_id = self.internal_id + # self.template_work_id = str(uuid.uuid4())[:8] + self.lifetime = lifetime + self.pending_time = pending_time + + if name: + self._name = name + "." + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + else: + self._name = 'idds.workflow.' + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + + if workload_id is None: + workload_id = int(time.time()) + self.workload_id = workload_id + + self.logger = logger + if self.logger is None: + self.setup_logger() + + self._works = {} + self.works = {} + self.work_sequence = {} # order list + + self.terminated_works = [] + self.initial_works = [] + # if the primary initial_work is not set, it's the first initial work. + self.primary_initial_work = None + self.independent_works = [] + + self.first_initial = False + self.new_to_run_works = [] + self.current_running_works = [] + + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + self.num_total_works = 0 + + self.last_work = None + + self.last_updated_at = datetime.datetime.utcnow() + self.expired = False + + self.to_update_transforms = {} + + # user defined Condition class + self.user_defined_conditions = {} + + self.username = None + self.userdn = None + self.proxy = None + + self._loop_condition_position = 'end' + self.loop_condition = None + + self.num_run = None + + """ + self._running_data_names = [] + for name in ['internal_id', 'template_work_id', 'workload_id', 'work_sequence', 'terminated_works', + 'first_initial', 'new_to_run_works', 'current_running_works', + 'num_subfinished_works', 'num_finished_works', 'num_failed_works', 'num_cancelled_works', 'num_suspended_works', + 'num_expired_works', 'num_total_works', 'last_work']: + self._running_data_names.append(name) + for name in ['works']: + self._running_data_names.append(name) + """ + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + + def get_template_work_id(self): + return self.template_work_id + + def get_template_id(self): + return self.template_work_id + + @property + def workload_id(self): + return self.get_metadata_item('workload_id') + + @workload_id.setter + def workload_id(self, value): + self.add_metadata_item('workload_id', value) + + @property + def lifetime(self): + # return self.get_metadata_item('lifetime', None) + return getattr(self, '_lifetime', None) + + @lifetime.setter + def lifetime(self, value): + # self.add_metadata_item('lifetime', value) + self._lifetime = value + + @property + def pending_time(self): + # return self.get_metadata_item('pending_time', None) + return getattr(self, '_pending_time', None) + + @pending_time.setter + def pending_time(self, value): + # self.add_metadata_item('pending_time', value) + self._pending_time = value + + @property + def last_updated_at(self): + last_updated_at = self.get_metadata_item('last_updated_at', None) + if last_updated_at and type(last_updated_at) in [str]: + last_updated_at = str_to_date(last_updated_at) + return last_updated_at + + @last_updated_at.setter + def last_updated_at(self, value): + self.add_metadata_item('last_updated_at', value) + + def has_new_updates(self): + self.last_updated_at = datetime.datetime.utcnow() + + @property + def expired(self): + t = self.get_metadata_item('expired', False) + if type(t) in [bool]: + return t + elif type(t) in [str] and t.lower() in ['true']: + return True + else: + return False + + @expired.setter + def expired(self, value): + self.add_metadata_item('expired', value) + + @property + def works(self): + return self._works + + @works.setter + def works(self, value): + self._works = value + work_metadata = {} + if self._works: + for k in self._works: + work = self._works[k] + if isinstance(work, Workflow): + work_metadata[k] = {'type': 'workflow', + 'metadata': work.metadata} + else: + work_metadata[k] = {'type': 'work', + 'work_id': work.work_id, + 'workload_id': work.workload_id, + 'status': work.status.value if work.status else work.status, + 'substatus': work.substatus.value if work.substatus else work.substatus, + 'transforming': work.transforming} + self.add_metadata_item('works', work_metadata) + + def refresh_works(self): + work_metadata = {} + if self._works: + for k in self._works: + work = self._works[k] + if isinstance(work, Workflow): + work.refresh_works() + work_metadata[k] = {'type': 'workflow', + 'metadata': work.metadata} + else: + work_metadata[k] = {'type': 'work', + 'work_id': work.work_id, + 'workload_id': work.workload_id, + 'status': work.status.value if work.status else work.status, + 'substatus': work.substatus.value if work.substatus else work.substatus, + 'transforming': work.transforming} + if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): + self.last_updated_at = work.last_updated_at + self.add_metadata_item('works', work_metadata) + + def load_works(self): + work_metadata = self.get_metadata_item('works', {}) + for k in self._works: + if k in work_metadata: + if work_metadata[k]['type'] == 'work': + self._works[k].work_id = work_metadata[k]['work_id'] + self._works[k].workload_id = work_metadata[k]['workload_id'] + self._works[k].transforming = work_metadata[k]['transforming'] + self._works[k].status = WorkStatus(work_metadata[k]['status']) if work_metadata[k]['status'] else work_metadata[k]['status'] + self._works[k].substatus = WorkStatus(work_metadata[k]['substatus']) if work_metadata[k]['substatus'] else work_metadata[k]['substatus'] + elif work_metadata[k]['type'] == 'workflow': + self._works[k].metadata = work_metadata[k]['metadata'] + + work = self._works[k] + if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): + self.last_updated_at = work.last_updated_at + + @property + def conditions(self): + return self._conditions + + @conditions.setter + def conditions(self, value): + self._conditions = value + conditions_metadata = {} + if self._conditions: + for k in self._conditions: + conditions_metadata[k] = self._conditions[k].metadata + self.add_metadata_item('conditions', conditions_metadata) + + @property + def work_conds(self): + return self._work_conds + + @work_conds.setter + def work_conds(self, value): + self._work_conds = value + # self.add_metadata_item('work_conds', value) + + def load_work_conditions(self): + conditions_metadata = self.get_metadata_item('conditions', {}) + for cond_internal_id in self._conditions: + if cond_internal_id in conditions_metadata: + self.conditions[cond_internal_id].metadata = conditions_metadata[cond_internal_id] + self.conditions[cond_internal_id].load_conditions(self.works) + + # work_conds = self.get_metadata_item('work_conds', {}) + # self._work_conds = work_conds + + @property + def loop_condition(self): + return self._loop_condition + + @loop_condition.setter + def loop_condition(self, value): + # self._loop_condition_position = position + self._loop_condition = value + if self._loop_condition: + self.add_metadata_item('loop_condition', self._loop_condition.get_condition_status()) + + @property + def work_sequence(self): + return self.get_metadata_item('work_sequence', {}) + + @work_sequence.setter + def work_sequence(self, value): + self.add_metadata_item('work_sequence', value) + + @property + def terminated_works(self): + return self.get_metadata_item('terminated_works', []) + + @terminated_works.setter + def terminated_works(self, value): + self.add_metadata_item('terminated_works', value) + + @property + def first_initial(self): + return self.get_metadata_item('first_initial', False) + + @first_initial.setter + def first_initial(self, value): + self.add_metadata_item('first_initial', value) + + @property + def new_to_run_works(self): + return self.get_metadata_item('new_to_run_works', []) + + @new_to_run_works.setter + def new_to_run_works(self, value): + self.add_metadata_item('new_to_run_works', value) + + @property + def current_running_works(self): + return self.get_metadata_item('current_running_works', []) + + @current_running_works.setter + def current_running_works(self, value): + self.add_metadata_item('current_running_works', value) + + @property + def num_subfinished_works(self): + return self.get_metadata_item('num_subfinished_works', 0) + + @num_subfinished_works.setter + def num_subfinished_works(self, value): + self.add_metadata_item('num_subfinished_works', value) + + @property + def num_finished_works(self): + return self.get_metadata_item('num_finished_works', 0) + + @num_finished_works.setter + def num_finished_works(self, value): + self.add_metadata_item('num_finished_works', value) + + @property + def num_failed_works(self): + return self.get_metadata_item('num_failed_works', 0) + + @num_failed_works.setter + def num_failed_works(self, value): + self.add_metadata_item('num_failed_works', value) + + @property + def num_cancelled_works(self): + return self.get_metadata_item('num_cancelled_works', 0) + + @num_cancelled_works.setter + def num_cancelled_works(self, value): + self.add_metadata_item('num_cancelled_works', value) + + @property + def num_suspended_works(self): + return self.get_metadata_item('num_suspended_works', 0) + + @num_suspended_works.setter + def num_suspended_works(self, value): + self.add_metadata_item('num_suspended_works', value) + + @property + def num_expired_works(self): + return self.get_metadata_item('num_expired_works', 0) + + @num_expired_works.setter + def num_expired_works(self, value): + self.add_metadata_item('num_expired_works', value) + + @property + def num_total_works(self): + return self.get_metadata_item('num_total_works', 0) + + @num_total_works.setter + def num_total_works(self, value): + self.add_metadata_item('num_total_works', value) + + @property + def last_work(self): + return self.get_metadata_item('last_work', None) + + @last_work.setter + def last_work(self, value): + self.add_metadata_item('last_work', value) + + @property + def to_update_transforms(self): + return self.get_metadata_item('to_update_transforms', {}) + + @to_update_transforms.setter + def to_update_transforms(self, value): + self.add_metadata_item('to_update_transforms', value) + + @property + def num_run(self): + return self.get_metadata_item('num_run', None) + + @num_run.setter + def num_run(self, value): + if value is not None: + self.add_metadata_item('num_run', value) + + def load_metadata(self): + self.load_works() + self.load_work_conditions() + self.load_parameter_links() + + def get_class_name(self): + return self.__class__.__name__ + + def setup_logger(self): + """ + Setup logger + """ + self.logger = logging.getLogger(self.get_class_name()) + + def log_info(self, info): + if self.logger is None: + self.setup_logger() + self.logger.info(info) + + def log_debug(self, info): + if self.logger is None: + self.setup_logger() + self.logger.debug(info) + + def get_internal_id(self): + return self.internal_id + + def copy(self): + new_wf = copy.deepcopy(self) + return new_wf + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + def get_works(self): + return self.works + + def get_new_work_to_run(self, work_id, new_parameters=None): + # 1. initialize works + # template_id = work.get_template_id() + work = self.works[work_id] + if isinstance(work, Workflow): + work.sync_works() + + work.sequence_id = self.num_total_works + work.parent_num_run = self.num_run + + works = self.works + self.works = works + # self.work_sequence.append(new_work.get_internal_id()) + self.work_sequence[str(self.num_total_works)] = work.get_internal_id() + self.num_total_works += 1 + self.new_to_run_works.append(work.get_internal_id()) + self.last_work = work.get_internal_id() + else: + if new_parameters: + work.set_parameters(new_parameters) + work.sequence_id = self.num_total_works + + work.initialize_work() + work.num_run = self.num_run + works = self.works + self.works = works + # self.work_sequence.append(new_work.get_internal_id()) + self.work_sequence[str(self.num_total_works)] = work.get_internal_id() + self.num_total_works += 1 + self.new_to_run_works.append(work.get_internal_id()) + self.last_work = work.get_internal_id() + + return work + + def register_user_defined_condition(self, condition): + cond_src = inspect.getsource(condition) + self.user_defined_conditions[condition.__name__] = cond_src + + def load_user_defined_condition(self): + # try: + # Condition() + # except NameError: + # global Condition + # import Condition + + for cond_src_name in self.user_defined_conditions: + # global cond_src_name + exec(self.user_defined_conditions[cond_src_name]) + + def set_workload_id(self, workload_id): + self.workload_id = workload_id + + def get_workload_id(self): + return self.workload_id + + def add_initial_works(self, work): + self.initial_works.append(work.get_internal_id()) + if self.primary_initial_work is None: + self.primary_initial_work = work.get_internal_id() + + def add_work(self, work, initial=False, primary=False): + self.first_initial = False + self.works[work.get_internal_id()] = work + if initial: + if primary: + self.primary_initial_work = work.get_internal_id() + self.add_initial_works(work) + + self.independent_works.append(work.get_internal_id()) + + def add_condition(self, cond): + self.first_initial = False + cond_works = cond.all_works() + for cond_work in cond_works: + assert(cond_work.get_internal_id() in self.get_works()) + + conditions = self.conditions + conditions[cond.get_internal_id()] = cond + self.conditions = conditions + + # if cond.current_work not in self.work_conds: + # self.work_conds[cond.current_work] = [] + # self.work_conds[cond.current_work].append(cond) + work_conds = self.work_conds + for work in cond.all_pre_works(): + if work.get_internal_id() not in work_conds: + work_conds[work.get_internal_id()] = [] + work_conds[work.get_internal_id()].append(cond.get_internal_id()) + self.work_conds = work_conds + + # if a work is a true_work or false_work of a condition, + # should remove it from independent_works + cond_next_works = cond.all_next_works() + for next_work in cond_next_works: + if next_work.get_internal_id() in self.independent_works: + self.independent_works.remove(next_work.get_internal_id()) + + def find_workflow_from_work(self, work): + if work.get_internal_id() in self._works: + return self + else: + for k in self._works: + wk = self._works[k] + if isinstance(wk, Workflow): + wf = wk.find_workflow_from_work(work) + if wf: + return wf + return None + + def add_parameter_link(self, work_source, work_destinations, parameter_link): + wf_s = self.find_workflow_from_work(work_source) + if not wf_s: + raise Exception("Cannot find work %s in the workflow." % work_source.get_internal_id()) + if work_source.get_internal_id() not in wf_s.parameter_links_source: + wf_s.parameter_links_source[work_source.get_internal_id()] = [] + wf_s.parameter_links_source[work_source.get_internal_id()].append(parameter_link.get_internal_id()) + + if type(work_destinations) not in [list, tuple]: + work_destinations = [work_destinations] + for work_destination in work_destinations: + wf = self.find_workflow_from_work(work_destination) + if not wf: + raise Exception("Cannot find work %s in the workflow." % work_destination.get_internal_id()) + if parameter_link.get_internal_id() not in wf.parameter_links: + wf.parameter_links[parameter_link.get_internal_id()] = parameter_link + if work_destination.get_internal_id() not in wf.parameter_links_destination: + wf.parameter_links_destination[work_destination.get_internal_id()] = [] + wf.parameter_links_destination[work_destination.get_internal_id()].append(parameter_link.get_internal_id()) + + def find_parameter_links_from_id(self, internal_id): + rets = [] + if internal_id in self.parameter_links: + rets.append((self, self.parameter_links[internal_id])) + for k in self._works: + wk = self._works[k] + if isinstance(wk, Workflow): + links = wk.find_parameter_links_from_id(internal_id) + rets = rets + links + return rets + + def refresh_parameter_links(self): + p_metadata = {} + for internal_id in self.parameter_links: + p_metadata[internal_id] = self.parameter_links[internal_id].metadata + self.add_metadata_item('parameter_links', p_metadata) + + def set_source_parameters(self, internal_id): + work = self.works[internal_id] + if internal_id in self.parameter_links_source: + for p_id in self.parameter_links_source[internal_id]: + p_links = self.find_parameter_links_from_id(p_id) + for wf, p_link in p_links: + p_link.set_parameters(work) + wf.refresh_parameter_links() + + def get_destination_parameters(self, internal_id): + # work = self.works[internal_id] + parameters = {} + if internal_id in self.parameter_links_destination: + for p_id in self.parameter_links_destination[internal_id]: + p_link = self.parameter_links[p_id] + parameters.update(p_link.get_parameters()) + return parameters + + def load_parameter_links(self): + p_metadata = self.get_metadata_item('parameter_links', {}) + for p_id in self.parameter_links: + if p_id in p_metadata: + self.parameter_links[p_id].metadata = p_metadata[p_id] + + def enable_next_works(self, work, cond): + self.log_debug("Checking Work %s condition: %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + # load_conditions should cover it. + # if cond and self.is_class_method(cond.cond): + # # cond_work_id = self.works[cond.cond['idds_method_class_id']] + # cond.cond = getattr(work, cond.cond['idds_method']) + + self.log_info("Work %s condition: %s" % (work.get_internal_id(), cond.conditions)) + next_works = cond.get_next_works(trigger=ConditionTrigger.ToTrigger) + self.log_info("Work %s condition status %s" % (work.get_internal_id(), cond.get_cond_status())) + self.log_info("Work %s next works %s" % (work.get_internal_id(), str(next_works))) + new_next_works = [] + if next_works is not None: + for next_work in next_works: + parameters = self.get_destination_parameters(next_work.get_internal_id()) + new_next_work = self.get_new_work_to_run(next_work.get_internal_id(), parameters) + work.add_next_work(new_next_work.get_internal_id()) + # cond.add_condition_work(new_next_work) ####### TODO: + new_next_works.append(new_next_work) + return new_next_works + + def add_loop_condition(self, condition, position='end'): + self.loop_condition_position = position + self.loop_condition = condition + + def has_loop_condition(self): + if self.loop_condition: + return True + return False + + def get_loop_condition_status(self): + if self.has_loop_condition(): + self.loop_condition.load_conditions(self.works) + return self.loop_condition.get_condition_status() + return False + + def __str__(self): + return str(json_dumps(self)) + + def get_new_works(self): + """ + *** Function called by Marshaller agent. + + new works to be ready to start + """ + self.sync_works() + works = [] + for k in self.new_to_run_works: + if isinstance(self.works[k], Work): + works.append(self.works[k]) + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_new_works() + for k in self.current_running_works: + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_new_works() + return works + + def get_current_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + works = [] + for k in self.current_running_works: + if isinstance(self.works[k], Work): + works.append(self.works[k]) + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_current_works() + return works + + def get_all_works(self): + """ + *** Function called by Marshaller agent. + + Current running works + """ + self.sync_works() + + works = [] + for k in self.works: + if isinstance(self.works[k], Work): + works.append(self.works[k]) + if isinstance(self.works[k], Workflow): + works = works + self.works[k].get_all_works() + return works + + def get_primary_initial_collection(self): + """ + *** Function called by Clerk agent. + """ + + if self.primary_initial_work: + return self.get_works()[self.primary_initial_work].get_primary_input_collection() + elif self.initial_works: + return self.get_works()[self.initial_works[0]].get_primary_input_collection() + elif self.independent_works: + return self.get_works()[self.independent_works[0]].get_primary_input_collection() + else: + keys = self.get_works().keys() + return self.get_works()[keys[0]].get_primary_input_collection() + return None + + def get_dependency_works(self, work_id, depth, max_depth): + if depth > max_depth: + return [] + + deps = [] + for dep_work_id in self.work_dependencies[work_id]: + deps.append(dep_work_id) + l_deps = self.get_dependency_works(dep_work_id, depth + 1, max_depth) + deps += l_deps + deps = list(dict.fromkeys(deps)) + return deps + + def order_independent_works(self): + ind_work_ids = self.independent_works + self.independent_works = [] + self.work_dependencies = {} + for ind_work_id in ind_work_ids: + work = self.works[ind_work_id] + self.work_dependencies[ind_work_id] = [] + for ind_work_id1 in ind_work_ids: + if ind_work_id == ind_work_id1: + continue + work1 = self.works[ind_work_id1] + if work.depend_on(work1): + self.work_dependencies[ind_work_id].append(ind_work_id1) + self.log_debug('work dependencies 1: %s' % str(self.work_dependencies)) + + max_depth = len(ind_work_ids) + 1 + work_dependencies = copy.deepcopy(self.work_dependencies) + for work_id in work_dependencies: + deps = self.get_dependency_works(work_id, 0, max_depth) + self.work_dependencies[work_id] = deps + self.log_debug('work dependencies 2: %s' % str(self.work_dependencies)) + + while True: + for work_id in self.work_dependencies: + if work_id not in self.independent_works and len(self.work_dependencies[work_id]) == 0: + self.independent_works.append(work_id) + for work_id in self.independent_works: + if work_id in self.work_dependencies: + del self.work_dependencies[work_id] + for work_id in self.work_dependencies: + for in_work_id in self.independent_works: + if in_work_id in self.work_dependencies[work_id]: + self.work_dependencies[work_id].remove(in_work_id) + if not self.work_dependencies: + break + self.log_debug('independent_works: %s' % str(self.independent_works)) + + def first_initialize(self): + # set new_to_run works + if not self.first_initial: + self.first_initial = True + self.order_independent_works() + if self.initial_works: + tostart_works = self.initial_works + elif self.independent_works: + tostart_works = self.independent_works + else: + tostart_works = list(self.get_works().keys()) + tostart_works = [tostart_works[0]] + + for work_id in tostart_works: + self.get_new_work_to_run(work_id) + + def sync_works(self): + self.first_initialize() + + self.refresh_works() + + for k in self.works: + work = self.works[k] + self.log_debug("work %s is_terminated(%s:%s)" % (work.get_internal_id(), work.is_terminated(), work.get_status())) + + for work in [self.works[k] for k in self.new_to_run_works]: + if work.transforming: + self.new_to_run_works.remove(work.get_internal_id()) + self.current_running_works.append(work.get_internal_id()) + + for work in [self.works[k] for k in self.current_running_works]: + if isinstance(work, Workflow): + work.sync_works() + + if work.is_terminated(): + self.set_source_parameters(work.get_internal_id()) + + if work.get_internal_id() in self.work_conds: + self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + json_dumps(self.work_conds[work.get_internal_id()], sort_keys=True, indent=4))) + for cond_id in self.work_conds[work.get_internal_id()]: + cond = self.conditions[cond_id] + self.log_debug("Work %s has condition dependencie %s" % (work.get_internal_id(), + json_dumps(cond, sort_keys=True, indent=4))) + self.enable_next_works(work, cond) + + if work.is_terminated(): + self.log_info("Work %s is terminated(%s)" % (work.get_internal_id(), work.get_status())) + self.log_debug("Work conditions: %s" % json_dumps(self.work_conds, sort_keys=True, indent=4)) + if work.get_internal_id() not in self.work_conds: + # has no next work + self.log_info("Work %s has no condition dependencies" % work.get_internal_id()) + self.terminated_works.append(work.get_internal_id()) + self.current_running_works.remove(work.get_internal_id()) + else: + # self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), + # json_dumps(self.work_conds[work.get_template_id()], sort_keys=True, indent=4))) + # for cond in self.work_conds[work.get_template_id()]: + # self.enable_next_works(work, cond) + self.terminated_works.append(work.get_internal_id()) + self.current_running_works.remove(work.get_internal_id()) + + if work.is_finished(): + self.num_finished_works += 1 + elif work.is_subfinished(): + self.num_subfinished_works += 1 + elif work.is_failed(): + self.num_failed_works += 1 + elif work.is_expired(): + self.num_expired_works += 1 + elif work.is_cancelled(): + self.num_cancelled_works += 1 + elif work.is_suspended(): + self.num_suspended_works += 1 + + # if work.is_terminated(): + # # if it's a loop workflow, to generate new loop + # if isinstance(work, Workflow): + # work.sync_works() + log_str = "num_total_works: %s" % self.num_total_works + log_str += ", num_finished_works: %s" % self.num_finished_works + log_str += ", num_subfinished_works: %s" % self.num_subfinished_works + log_str += ", num_failed_works: %s" % self.num_failed_works + log_str += ", num_expired_works: %s" % self.num_expired_works + log_str += ", num_cancelled_works: %s" % self.num_cancelled_works + log_str += ", num_suspended_works: %s" % self.num_suspended_works + self.log_debug(log_str) + + def resume_works(self): + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + + self.last_updated_at = datetime.datetime.utcnow() + + t_works = self.terminated_works + self.terminated_works = [] + self.current_running_works = self.current_running_works + t_works + for work in [self.works[k] for k in self.current_running_works]: + if isinstance(work, Workflow): + work.resume_works() + else: + work.resume_work() + + def clean_works(self): + self.num_subfinished_works = 0 + self.num_finished_works = 0 + self.num_failed_works = 0 + self.num_cancelled_works = 0 + self.num_suspended_works = 0 + self.num_expired_works = 0 + self.num_total_works = 0 + + self.last_updated_at = datetime.datetime.utcnow() + + self.terminated_works = [] + self.current_running_works = [] + self.works = {} + self.work_sequence = {} # order list + + self.first_initial = False + self.new_to_run_works = [] + + def get_exact_workflows(self): + """ + *** Function called by Clerk agent. + + TODO: The primary dataset for the initial work is a dataset with '*'. + workflow.primary_initial_collection = 'some datasets with *' + collections = get_collection(workflow.primary_initial_collection) + wfs = [] + for coll in collections: + wf = self.copy() + wf.name = self.name + "_" + number + wf.primary_initial_collection = coll + wfs.append(wf) + return wfs + """ + return [self] + + def is_terminated(self): + """ + *** Function called by Marshaller agent. + """ + self.sync_works() + if len(self.new_to_run_works) == 0 and len(self.current_running_works) == 0: + return True + return False + + def is_finished(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and self.num_finished_works == self.num_total_works + + def is_subfinished(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_finished_works + self.num_subfinished_works > 0 and self.num_finished_works + self.num_subfinished_works <= self.num_total_works) + + def is_failed(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_failed_works > 0) and (self.num_cancelled_works == 0) and (self.num_suspended_works == 0) and (self.num_expired_works == 0) + + def is_to_expire(self, expired_at=None, pending_time=None, request_id=None): + if self.expired: + # it's already expired. avoid sending duplicated messages again and again. + return False + if expired_at: + if type(expired_at) in [str]: + expired_at = str_to_date(expired_at) + if expired_at < datetime.datetime.utcnow(): + self.logger.info("Request(%s) expired_at(%s) is smaller than utc now(%s), expiring" % (request_id, + expired_at, + datetime.datetime.utcnow())) + return True + + act_pending_time = None + if self.pending_time: + # in days + act_pending_time = float(self.pending_time) + else: + if pending_time: + act_pending_time = float(pending_time) + if act_pending_time: + act_pending_seconds = int(86400 * act_pending_time) + if self.last_updated_at + datetime.timedelta(seconds=act_pending_seconds) < datetime.datetime.utcnow(): + log_str = "Request(%s) last updated at(%s) + pending seconds(%s)" % (request_id, + self.last_updated_at, + act_pending_seconds) + log_str += " is smaller than utc now(%s), expiring" % (datetime.datetime.utcnow()) + self.logger.info(log_str) + return True + + return False + + def is_expired(self): + """ + *** Function called by Marshaller agent. + """ + # return self.is_terminated() and (self.num_expired_works > 0) + return self.is_terminated() and self.expired + + def is_cancelled(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_cancelled_works > 0) + + def is_suspended(self): + """ + *** Function called by Marshaller agent. + """ + return self.is_terminated() and (self.num_suspended_works > 0) + + def get_terminated_msg(self): + """ + *** Function called by Marshaller agent. + """ + if self.last_work: + return self.works[self.last_work].get_terminated_msg() + return None + + def get_status(self): + if self.is_terminated(): + if self.is_finished(): + return WorkStatus.Finished + elif self.is_subfinished(): + return WorkStatus.SubFinished + elif self.is_failed(): + return WorkStatus.Failed + elif self.is_expired(): + return WorkStatus.Expired + elif self.is_cancelled(): + return WorkStatus.Cancelled + elif self.is_suspended(): + return WorkStatus.Suspended + return WorkStatus.Transforming + + def depend_on(self, work): + return False + + def add_proxy(self): + self.proxy = get_proxy() + if not self.proxy: + raise Exception("Cannot get local proxy") + + def get_proxy(self): + return self.proxy + + +class Workflow(Base): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # super(Workflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + self.logger = logger + if self.logger is None: + self.setup_logger() + + self.template = WorkflowBase(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + self.parent_num_run = None + self._num_run = 0 + self.runs = {} + self.loop_condition_position = 'end' + + def setup_logger(self): + # Setup logger + self.logger = logging.getLogger(self.get_class_name()) + + def __deepcopy__(self, memo): + logger = self.logger + self.logger = None + + cls = self.__class__ + result = cls.__new__(cls) + + memo[id(self)] = result + + # Deep copy all other attributes + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + + self.logger = logger + result.logger = logger + return result + + @property + def metadata(self): + run_metadata = {'parent_num_run': self.parent_num_run, + 'num_run': self._num_run, + 'runs': {}} + for run_id in self.runs: + run_metadata['runs'][run_id] = self.runs[run_id].metadata + return run_metadata + + @metadata.setter + def metadata(self, value): + run_metadata = value + self.parent_num_run = run_metadata['parent_num_run'] + self._num_run = run_metadata['num_run'] + runs = run_metadata['runs'] + for run_id in runs: + self.runs[run_id] = self.template.copy() + self.runs[run_id].metadata = runs[run_id] + # self.add_metadata_item('runs', ) + + @property + def independent_works(self): + if self.runs: + return self.runs[str(self.num_run)].independent_works + return self.template.independent_works + + @independent_works.setter + def independent_works(self, value): + if self.runs: + self.runs[str(self.num_run)].independent_works = value + self.template.independent_works = value + + @property + def last_updated_at(self): + if self.runs: + return self.runs[str(self.num_run)].last_updated_at + return None + + @last_updated_at.setter + def last_updated_at(self, value): + if self.runs: + self.runs[str(self.num_run)].last_updated_at = value + + @property + def num_run(self): + if self.parent_num_run: + return self.parent_num_run * 100 + self._num_run + return self._num_run + + @num_run.setter + def num_run(self, value): + if self.parent_num_run: + self._num_run = value - self.parent_num_run * 100 + else: + self._num_run = value + + @property + def transforming(self): + if self.runs and str(self.num_run) in self.runs: + return True + return False + + @transforming.setter + def transforming(self, value): + if self._num_run < 1: + self._num_run = 1 + if str(self.num_run) not in self.runs: + self.runs[str(self.num_run)] = self.template.copy() + if self.runs[str(self.num_run)].has_loop_condition(): + self.runs[str(self.num_run)]._num_run = self.num_run + if self._num_run > 1: + p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') + self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) + + def set_workload_id(self, workload_id): + if self.runs: + self.runs[str(self.num_run)].workload_id = workload_id + else: + self.template.workload_id = workload_id + # self.dynamic.workload_id = workload_id + + def get_internal_id(self): + if self.runs: + return self.runs[str(self.num_run)].get_internal_id() + return self.template.get_internal_id() + + def get_workload_id(self): + if self.runs: + return self.runs[str(self.num_run)].workload_id + return self.template.workload_id + + def add_work(self, work, initial=False, primary=False): + self.template.add_work(work, initial, primary) + + def add_condition(self, cond): + self.template.add_condition(cond) + + def add_parameter_link(self, work_source, work_destinations, parameter_link): + self.template.add_parameter_link(work_source, work_destinations, parameter_link) + + def find_workflow_from_work(self, work): + return self.template.find_workflow_from_work(work) + + def find_parameter_links_from_id(self, internal_id): + if self.runs: + return self.runs[str(self.num_run)].find_parameter_links_from_id(internal_id) + return [] + + def refresh_parameter_links(self): + if self.runs: + self.runs[str(self.num_run)].refresh_parameter_links() + + def get_new_works(self): + self.sync_works() + if self.runs: + return self.runs[str(self.num_run)].get_new_works() + return [] + + def get_current_works(self): + self.sync_works() + if self.runs: + return self.runs[str(self.num_run)].get_current_works() + return [] + + def get_all_works(self): + self.sync_works() + if self.runs: + return self.runs[str(self.num_run)].get_all_works() + return [] + + def get_primary_initial_collection(self): + if self.runs: + return self.runs[str(self.num_run)].get_primary_initial_collection() + return self.template.get_primary_initial_collection() + + def resume_works(self): + if self.runs: + self.runs[str(self.num_run)].resume_works() + + def clean_works(self): + if self.runs: + self.runs[str(self.num_run)].clean_works() + + def is_terminated(self): + if self.runs: + if self.runs[str(self.num_run)].is_terminated(): + if not self.runs[str(self.num_run)].has_loop_condition() or not self.runs[str(self.num_run)].get_loop_condition_status(): + return True + return False + + def is_finished(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_finished() + return False + + def is_subfinished(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_subfinished() + return False + + def is_failed(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_failed() + return False + + def is_expired(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_expired() + return False + + def is_cancelled(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_cancelled() + return False + + def is_suspended(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].is_suspended() + return False + + def get_terminated_msg(self): + if self.is_terminated(): + return self.runs[str(self.num_run)].get_terminated_msg() + return None + + def get_status(self): + if not self.runs: + return WorkStatus.New + if not self.is_terminated(): + return WorkStatus.Transforming + return self.runs[str(self.num_run)].get_status() + + def depend_on(self, work): + return self.template.depend_on(work) + + def add_proxy(self): + self.template.add_proxy() + + def get_proxy(self): + self.template.get_proxy() + + def add_loop_condition(self, condition, position='end'): + if not position or position != 'begin': + position = 'end' + position = 'end' # force position to end currently. position = 'begin' is not supported now. + self.template.add_loop_condition(condition, position=position) + self.loop_condition_position = position + + def refresh_works(self): + if self.runs: + self.runs[str(self.num_run)].refresh_works() + + def sync_works(self): + # position is end. + if self._num_run < 1: + self._num_run = 1 + if str(self.num_run) not in self.runs: + self.runs[str(self.num_run)] = self.template.copy() + if self.runs[str(self.num_run)].has_loop_condition(): + self.runs[str(self.num_run)]._num_run = self._num_run + if self._num_run > 1: + p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') + self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) + + self.runs[str(self.num_run)].sync_works() + + if self.runs[str(self.num_run)].is_terminated(): + if self.runs[str(self.num_run)].has_loop_condition(): + if self.runs[str(self.num_run)].get_loop_condition_status(): + self._num_run += 1 + self.runs[str(self.num_run)] = self.template.copy() + + self.runs[str(self.num_run)]._num_run = self._num_run + p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') + self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) + + +class SubWorkflow(Workflow): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # Init a workflow. + super(SubWorkflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) + + +class LoopWorkflow(Workflow): + def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): + # Init a workflow. + super(LoopWorkflow, self).__init__(name=name, workload_id=workload_id, lifetime=lifetime, pending_time=pending_time, logger=logger) From 82d428a94ea374a56daaeec6139769dce669859b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 21:44:45 +0200 Subject: [PATCH 085/156] fix set parameters --- workflow/lib/idds/workflowv2/work.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 8a2dfdb5..b232540d 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -134,6 +134,9 @@ def collection(self, value): self.status = self._collection['status'] self.substatus = self._collection['substatus'] + def to_origin_dict(self): + return {'scope': self.scope, 'name': self.name} + class Processing(Base): @@ -890,10 +893,11 @@ def num_run(self): def num_run(self, value): if value is not None: self.add_metadata_item('num_run', value) - for k in self._collections: - coll = self._collections[k] - if type(coll) in [Collection]: - coll.name = coll.name + "." + str(value) + if value > 1: + # for k in self._collections: + for coll in self.output_collections: + if type(coll) in [Collection]: + coll.name = coll.name + "." + str(value) @property def primary_input_collection(self): @@ -1085,8 +1089,9 @@ def set_parameters(self, parameters): self.parameters = parameters for p in self.parameters: if self.parameters[p] is not None and hasattr(self, p): - fp = getattr(self, p) - fp = self.parameters[p] # noqa F841 + # fp = getattr(self, p) + # fp = self.parameters[p] # noqa F841 + setattr(self, p, self.parameters[p]) def get_parameters(self): return self.parameters From 19ab5161b9914d297cc59100f35832867e146d6a Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 21:45:10 +0200 Subject: [PATCH 086/156] fix parameter links --- workflow/lib/idds/workflowv2/workflow.py | 53 +++++++++++++++++++----- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 26bb72d5..081de954 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -22,7 +22,7 @@ from idds.common.utils import json_dumps, setup_logging, get_proxy from idds.common.utils import str_to_date from .base import Base -from .work import Work +from .work import Work, Collection setup_logging(__name__) @@ -483,6 +483,7 @@ def add_condition(self, cond): class ParameterLink(Base): def __init__(self, parameters): + super(ParameterLink, self).__init__() self.parameters = {} self.num_parameters = 0 if parameters: @@ -506,17 +507,21 @@ def get_internal_id(self): return self.internal_id def get_parameter_value(self, work, p): + ret = None p_f = getattr(work, p, 'None') if p_f: if callable(p_f): - return p_f() + ret = p_f() else: - return p_f + ret = p_f else: - return None + ret = None + if ret and type(ret) in [Collection] and hasattr(ret, 'to_origin_dict'): + ret = ret.to_origin_dict() + return ret def set_parameters(self, work): - p_values = [] + p_values = {} for p in self.parameters: p_values[p] = self.get_parameter_value(work, self.parameters[p]['source']) self.add_metadata_item('parameters', p_values) @@ -527,6 +532,7 @@ def get_parameters(self): for p in self.parameters: if p in p_values: ret[self.parameters[p]['destination']] = p_values[p] + return ret class WorkflowBase(Base): @@ -979,6 +985,7 @@ def get_new_work_to_run(self, work_id, new_parameters=None): self.new_to_run_works.append(work.get_internal_id()) self.last_work = work.get_internal_id() else: + new_parameters = self.get_destination_parameters(work_id) if new_parameters: work.set_parameters(new_parameters) work.sequence_id = self.num_total_works @@ -1107,11 +1114,30 @@ def refresh_parameter_links(self): p_metadata[internal_id] = self.parameter_links[internal_id].metadata self.add_metadata_item('parameter_links', p_metadata) + def get_parameter_links_metadata(self): + p_metadata = {} + for internal_id in self.parameter_links: + p_metadata[internal_id] = self.parameter_links[internal_id].metadata + self.add_metadata_item('parameter_links', p_metadata) + return p_metadata + + def set_parameter_links_metadata(self, p_links): + for internal_id in self.parameter_links: + if internal_id in p_links: + p_metadata = p_links[internal_id] + self.parameter_links[internal_id].metadata = p_metadata + def set_source_parameters(self, internal_id): work = self.works[internal_id] + # if type(work) in [Work]: + # print(work.work_id) + # print(internal_id) + # print(self.parameter_links_source) if internal_id in self.parameter_links_source: for p_id in self.parameter_links_source[internal_id]: + # print(p_id) p_links = self.find_parameter_links_from_id(p_id) + # print(p_links) for wf, p_link in p_links: p_link.set_parameters(work) wf.refresh_parameter_links() @@ -1146,8 +1172,8 @@ def enable_next_works(self, work, cond): new_next_works = [] if next_works is not None: for next_work in next_works: - parameters = self.get_destination_parameters(next_work.get_internal_id()) - new_next_work = self.get_new_work_to_run(next_work.get_internal_id(), parameters) + # parameters = self.get_destination_parameters(next_work.get_internal_id()) + new_next_work = self.get_new_work_to_run(next_work.get_internal_id()) work.add_next_work(new_next_work.get_internal_id()) # cond.add_condition_work(new_next_work) ####### TODO: new_next_works.append(new_next_work) @@ -1583,6 +1609,8 @@ def metadata(self): 'runs': {}} for run_id in self.runs: run_metadata['runs'][run_id] = self.runs[run_id].metadata + if not self.runs: + run_metadata['parameter_links'] = self.template.get_parameter_links_metadata() return run_metadata @metadata.setter @@ -1591,6 +1619,9 @@ def metadata(self, value): self.parent_num_run = run_metadata['parent_num_run'] self._num_run = run_metadata['num_run'] runs = run_metadata['runs'] + if not runs and 'parameter_links' in run_metadata: + parameter_links = run_metadata['parameter_links'] + self.template.set_parameter_links_metadata(parameter_links) for run_id in runs: self.runs[run_id] = self.template.copy() self.runs[run_id].metadata = runs[run_id] @@ -1645,7 +1676,7 @@ def transforming(self, value): if str(self.num_run) not in self.runs: self.runs[str(self.num_run)] = self.template.copy() if self.runs[str(self.num_run)].has_loop_condition(): - self.runs[str(self.num_run)]._num_run = self.num_run + self.runs[str(self.num_run)].num_run = self.num_run if self._num_run > 1: p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) @@ -1682,7 +1713,7 @@ def find_workflow_from_work(self, work): def find_parameter_links_from_id(self, internal_id): if self.runs: return self.runs[str(self.num_run)].find_parameter_links_from_id(internal_id) - return [] + return self.template.find_parameter_links_from_id(internal_id) def refresh_parameter_links(self): if self.runs: @@ -1795,7 +1826,7 @@ def sync_works(self): if str(self.num_run) not in self.runs: self.runs[str(self.num_run)] = self.template.copy() if self.runs[str(self.num_run)].has_loop_condition(): - self.runs[str(self.num_run)]._num_run = self._num_run + self.runs[str(self.num_run)].num_run = self._num_run if self._num_run > 1: p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) @@ -1808,7 +1839,7 @@ def sync_works(self): self._num_run += 1 self.runs[str(self.num_run)] = self.template.copy() - self.runs[str(self.num_run)]._num_run = self._num_run + self.runs[str(self.num_run)].num_run = self._num_run p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) From e115a8cd962e311635b22cd79aa15b9973bd80f9 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 20 Oct 2021 21:46:23 +0200 Subject: [PATCH 087/156] add new workflow tests --- .../idds/tests/test_workflow_condition_v2.py | 1332 +++++++++++++++++ 1 file changed, 1332 insertions(+) create mode 100644 main/lib/idds/tests/test_workflow_condition_v2.py diff --git a/main/lib/idds/tests/test_workflow_condition_v2.py b/main/lib/idds/tests/test_workflow_condition_v2.py new file mode 100644 index 00000000..3dc11784 --- /dev/null +++ b/main/lib/idds/tests/test_workflow_condition_v2.py @@ -0,0 +1,1332 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 + + +""" +Test workflow condtions. +""" + +import unittest2 as unittest +# from nose.tools import assert_equal +from idds.common.utils import setup_logging + +from idds.common.utils import json_dumps, json_loads + +from idds.workflowv2.work import Work, WorkStatus +from idds.workflowv2.workflow import (CompositeCondition, AndCondition, OrCondition, + Condition, ConditionTrigger, Workflow, ParameterLink) + + +setup_logging(__name__) + + +class TestWorkflowCondtion(unittest.TestCase): + + def test_condition(self): + # init_p = Parameter({'input_dataset': 'data17:data17.test.raw.1'}) + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) + work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) + work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) + work7 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=7, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) + work8 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=8, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) + + workflow = Workflow() + workflow.add_work(work1, initial=True) + workflow.add_work(work2, initial=True) + workflow.add_work(work3, initial=False) + workflow.add_work(work8, initial=False) + + # CompositeCondition + cond1 = CompositeCondition(conditions=work1.is_finished, true_works=work2, false_works=work3) + works = cond1.all_works() + assert(works == [work1, work2, work3]) + works = cond1.all_pre_works() + assert(works == [work1]) + works = cond1.all_next_works() + assert(works == [work2, work3]) + cond_status = cond1.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond1.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + + works = cond1.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond1.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work3]) + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work2]) + works = cond1.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + + works = cond1.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond1.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + # CompositeCondition + cond2 = CompositeCondition(conditions=[work1.is_finished, work2.is_finished, work3.is_finished], true_works=[work4, work5], false_works=[work6, work7]) + + works = cond2.all_works() + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond2.all_pre_works() + assert(works == [work1, work2, work3]) + works = cond2.all_next_works() + assert(works == [work4, work5, work6, work7]) + cond_status = cond2.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond2.get_condition_status() + assert(cond_status is False) + work2.status = WorkStatus.Finished + cond_status = cond2.get_condition_status() + assert(cond_status is False) + work3.status = WorkStatus.Finished + cond_status = cond2.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond2.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond2.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work6, work7]) + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work4, work5]) + works = cond2.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond2.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond2.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + # AndCondition + cond3 = AndCondition(conditions=[work1.is_finished, work2.is_finished, work3.is_finished], true_works=[work4, work5], false_works=[work6, work7]) + + works = cond3.all_works() + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond3.all_pre_works() + assert(works == [work1, work2, work3]) + works = cond3.all_next_works() + assert(works == [work4, work5, work6, work7]) + cond_status = cond3.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond3.get_condition_status() + assert(cond_status is False) + work2.status = WorkStatus.Finished + cond_status = cond3.get_condition_status() + assert(cond_status is False) + work3.status = WorkStatus.Finished + cond_status = cond3.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond3.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond3.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work6, work7]) + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work4, work5]) + works = cond3.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond3.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + work2.status = WorkStatus.Finished + work3.status = WorkStatus.Finished + works = cond3.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + # OrCondtion + cond4 = OrCondition(conditions=[work1.is_finished, work2.is_finished, work3.is_finished], true_works=[work4, work5], false_works=[work6, work7]) + + works = cond4.all_works() + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond4.all_pre_works() + assert(works == [work1, work2, work3]) + works = cond4.all_next_works() + assert(works == [work4, work5, work6, work7]) + cond_status = cond4.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond4.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.Finished + cond_status = cond4.get_condition_status() + assert(cond_status is True) + work2.status = WorkStatus.New + work3.status = WorkStatus.Finished + cond_status = cond4.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond4.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + # work2.status = WorkStatus.Finished + # work3.status = WorkStatus.Finished + works = cond4.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work6, work7]) + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + # work2.status = WorkStatus.Finished + # work3.status = WorkStatus.Finished + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work4, work5]) + works = cond4.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + works = cond4.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work6, work7]) + work1.status = WorkStatus.Finished + # work2.status = WorkStatus.Finished + # work3.status = WorkStatus.Finished + works = cond4.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work4, work5]) + work1.status = WorkStatus.New + work2.status = WorkStatus.New + work3.status = WorkStatus.New + + # Condition + cond5 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + + works = cond5.all_works() + assert(works == [work1, work2, work3]) + works = cond5.all_pre_works() + assert(works == [work1]) + works = cond5.all_next_works() + assert(works == [work2, work3]) + cond_status = cond5.get_condition_status() + assert(cond_status is False) + + work1.status = WorkStatus.Finished + cond_status = cond5.get_condition_status() + assert(cond_status is True) + work1.status = WorkStatus.New + + works = cond5.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond5.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work3]) + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work2]) + works = cond5.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.New + + works = cond5.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work3]) + work1.status = WorkStatus.Finished + works = cond5.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work2]) + work1.status = WorkStatus.New + + # multiple conditions + cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + + works = cond7.all_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond7.all_pre_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4, work5]) + works = cond7.all_next_works() + works.sort(key=lambda x: x.work_id) + # print([w.work_id for w in works]) + assert(works == [work2, work3, work6, work7]) + cond_status = cond7.get_condition_status() + assert(cond_status is False) + + work4.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is False) + work5.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is True) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + return workflow + + def print_workflow(self, workflow): + print('print workflow') + print(workflow.conditions) + for cond_id in workflow.conditions: + print(cond_id) + cond = workflow.conditions[cond_id] + print(cond) + print(cond.conditions) + print(cond.true_works) + print(cond.false_works) + for w in cond.true_works: + print(w) + if isinstance(w, CompositeCondition): + print(w.conditions) + print(w.true_works) + print(w.false_works) + + def test_workflow(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) + work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) + work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) + work7 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=7, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) + work8 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=8, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + workflow.add_work(work3, initial=False) + workflow.add_work(work4, initial=False) + workflow.add_work(work5, initial=False) + workflow.add_work(work6, initial=False) + workflow.add_work(work7, initial=False) + workflow.add_work(work8, initial=False) + + # multiple conditions + cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + + workflow.add_condition(cond7) + id_works = workflow.independent_works + # print(id_works) + id_works.sort() + id_works_1 = [work1, work4, work5, work8] + id_works_1 = [w.get_template_id() for w in id_works_1] + id_works_1.sort() + # id_works.sort(key=lambda x: x.work_id) + assert(id_works == id_works_1) + + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow1 = json_loads(workflow_str) + # print('before load_metadata') + # self.print_workflow(workflow1) + workflow1.load_metadata() + # print('after load_metadata') + # self.print_workflow(workflow1) + workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4) + assert(workflow_str == workflow_str1) + + works = cond7.all_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3, work4, work5, work6, work7]) + works = cond7.all_pre_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4, work5]) + works = cond7.all_next_works() + works.sort(key=lambda x: x.work_id) + # print([w.work_id for w in works]) + assert(works == [work2, work3, work6, work7]) + cond_status = cond7.get_condition_status() + assert(cond_status is False) + + work4.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is False) + work5.status = WorkStatus.Finished + cond_status = cond7.get_condition_status() + assert(cond_status is True) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + return workflow + + def test_workflow_condition_reload(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) + work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) + work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) + work7 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=7, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.raw.1'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work2'}]) + work8 = Work(executable='echo', + arguments='--in=IN_DATASET --out=OUT_DATASET', + sandbox=None, + work_id=8, + primary_input_collection={'scope': 'data17', 'name': 'data17.test.work2'}, + output_collections=[{'scope': 'data17', 'name': 'data17.test.work3'}]) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + workflow.add_work(work3, initial=False) + workflow.add_work(work4, initial=False) + workflow.add_work(work5, initial=False) + workflow.add_work(work6, initial=False) + workflow.add_work(work7, initial=False) + workflow.add_work(work8, initial=False) + + # multiple conditions + cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + + workflow.add_condition(cond7) + + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow1 = json_loads(workflow_str) + # print('before load_metadata') + # self.print_workflow(workflow1) + workflow1.load_metadata() + # print('after load_metadata') + # self.print_workflow(workflow1) + workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4) + assert(workflow_str == workflow_str1) + + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work5.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work3, work6]) + work1.status = WorkStatus.Finished + works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work2, work6]) + work4.status = WorkStatus.New + work5.status = WorkStatus.New + work1.status = WorkStatus.New + + return workflow + + def test_workflow_loop(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow.add_loop_condition(cond) + + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow1 = json_loads(workflow_str) + # print('before load_metadata') + # self.print_workflow(workflow1) + workflow1.load_metadata() + # print('after load_metadata') + # self.print_workflow(workflow1) + workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4) + assert(workflow_str == workflow_str1) + + def test_workflow_loop1(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow.add_loop_condition(cond) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.num_run == 1) + + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + return workflow + + def test_workflow_loop2(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow = Workflow() + workflow.add_work(work1, initial=False) + workflow.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow.add_loop_condition(cond) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.num_run == 1) + + for work in works: + work.transforming = True + work.status = WorkStatus.Finished + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.num_run == 2) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + + for work in works: + work.transforming = True + work.status = WorkStatus.Finished + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.num_run == 3) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + + for work in works: + work.transforming = True + work.status = WorkStatus.Failed + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.num_run == 3) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + + return workflow + + def test_workflow_subworkflow(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + + works = workflow.get_new_works() + # print(json_dumps(workflow, sort_keys=True, indent=4)) + # print(json_dumps(works, sort_keys=True, indent=4)) + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3]) + # assert(workflow1.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + assert(workflow.is_terminated() is True) + + def test_workflow_subworkflow1(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + cond = Condition(cond=work3.is_finished, true_work=workflow1) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + workflow.add_condition(cond) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work3]) + # assert(workflow.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.is_terminated() is True) + + def test_workflow_subworkflow2(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + cond = Condition(cond=work3.is_finished, true_work=workflow1) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + workflow.add_condition(cond) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work3]) + # assert(workflow.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.is_terminated() is False) + + for work in works: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.is_terminated() is True) + + def test_workflow_subloopworkflow(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3]) + # assert(workflow1.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + assert(workflow.is_terminated() is True) + + def test_workflow_subloopworkflow1(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2, work3]) + # assert(workflow1.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + assert(workflow.is_terminated() is False) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + # assert(workflow1.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + assert(workflow.is_terminated() is True) + + def test_workflow_subloopworkflow2(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + cond1 = Condition(cond=work3.is_finished, true_work=workflow1) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + workflow.add_condition(cond1) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work3]) + # assert(workflow.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.is_terminated() is True) + + def test_workflow_subloopworkflow3(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + cond1 = Condition(cond=work3.is_finished, true_work=workflow1) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + workflow.add_condition(cond1) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work3]) + # assert(workflow.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.is_terminated() is False) + + for work in works: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + assert(works == [work1, work2]) + assert(workflow.is_terminated() is False) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + assert(works == [work1, work2]) + assert(workflow.is_terminated() is False) + + for work in works: + work.transforming = True + work.status = WorkStatus.Failed + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.is_terminated() is True) + + def test_workflow_subloopworkflow_reload(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) + cond1 = Condition(cond=work3.is_finished, true_work=workflow1) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + workflow.add_condition(cond1) + + # reload + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow = json_loads(workflow_str) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work3]) + # assert(workflow.num_run == 1) + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + # reload + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow = json_loads(workflow_str) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work2]) + assert(workflow.is_terminated() is False) + + for work in works: + work.transforming = True + work.status = WorkStatus.Finished + + # reload + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow = json_loads(workflow_str) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + assert(works == [work1, work2]) + assert(workflow.is_terminated() is False) + + for work in works: + work.transforming = True + work.status = WorkStatus.Failed + + # reload + workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + workflow = json_loads(workflow_str) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.is_terminated() is True) + + def test_workflow_subloopworkflow_parameter_link(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_1'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_1'}) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_2'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_2'}) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond1 = Condition(cond=work1.is_finished, true_work=work2) + workflow1.add_condition(cond1) + + p_link = ParameterLink(parameters=[{'source': 'primary_output_collection', + 'destination': 'primary_input_collection'}]) + workflow1.add_parameter_link(work1, work2, p_link) + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1]) + # assert(workflow.num_run == 1) + work1_1 = works[0] + assert(work1_1.primary_input_collection.name == 'input_test_work_1') + assert(work1_1.primary_output_collection.name == 'output_test_work_1') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + assert(workflow1.is_terminated() is False) + work2_1 = works[0] + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + + assert(work2_1.primary_input_collection.name == 'output_test_work_1') + assert(work2_1.primary_output_collection.name == 'output_test_work_2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow1.is_terminated() is True) + + def test_workflow_subloopworkflow_parameter_link1(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_1'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_1'}) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_2'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_2'}) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond1 = Condition(cond=work1.is_finished, true_work=work2) + workflow1.add_condition(cond1) + + p_link = ParameterLink(parameters=[{'source': 'primary_output_collection', + 'destination': 'primary_input_collection'}]) + workflow1.add_parameter_link(work1, work2, p_link) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1]) + # assert(workflow.num_run == 1) + work1_1 = works[0] + assert(work1_1.primary_input_collection.name == 'input_test_work_1') + assert(work1_1.primary_output_collection.name == 'output_test_work_1') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + assert(workflow1.is_terminated() is False) + work2_1 = works[0] + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + + assert(work2_1.primary_input_collection.name == 'output_test_work_1') + assert(work2_1.primary_output_collection.name == 'output_test_work_2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1]) + assert(workflow1.is_terminated() is False) + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + work1_2 = works[0] + assert(work1_2.primary_input_collection.name == 'input_test_work_1') + assert(work1_2.primary_output_collection.name == 'output_test_work_1.2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + assert(workflow1.is_terminated() is False) + work2_2 = works[0] + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + + assert(work2_2.primary_input_collection.name == 'output_test_work_1.2') + assert(work2_2.primary_output_collection.name == 'output_test_work_2.2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + + works = workflow1.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow1.is_terminated() is True) + + def test_workflow_subloopworkflow_parameter_link2(self): + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_1'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_1'}) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_2'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_2'}) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + cond1 = Condition(cond=work1.is_finished, true_work=work2) + workflow1.add_condition(cond1) + + p_link = ParameterLink(parameters=[{'source': 'primary_output_collection', + 'destination': 'primary_input_collection'}]) + workflow1.add_parameter_link(work1, work2, p_link) + + cond = Condition(cond=work2.is_finished) + workflow1.add_loop_condition(cond) + + work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_3'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_3'}) + cond2 = Condition(cond=work3.is_finished, true_work=workflow1) + p_link1 = ParameterLink(parameters=[{'source': 'primary_output_collection', + 'destination': 'primary_input_collection'}]) + + workflow = Workflow() + workflow.add_work(work3, initial=False) + workflow.add_work(workflow1, initial=False) + workflow.add_condition(cond2) + workflow.add_parameter_link(work3, work1, p_link1) + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work3]) + # assert(workflow.num_run == 1) + work3_1 = works[0] + assert(work3_1.primary_input_collection.name == 'input_test_work_3') + assert(work3_1.primary_output_collection.name == 'output_test_work_3') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1]) + assert(workflow.is_terminated() is False) + work1_1 = works[0] + # workflow_str = json_dumps(workflow, sort_keys=True, indent=4) + # print(workflow_str) + + assert(work1_1.primary_input_collection.name == 'output_test_work_3') + assert(work1_1.primary_output_collection.name == 'output_test_work_1') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + assert(workflow1.is_terminated() is False) + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + work2_1 = works[0] + assert(work2_1.primary_input_collection.name == 'output_test_work_1') + assert(work2_1.primary_output_collection.name == 'output_test_work_2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1]) + assert(workflow1.is_terminated() is False) + work1_2 = works[0] + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + + assert(work1_2.primary_input_collection.name == 'output_test_work_3') + assert(work1_2.primary_output_collection.name == 'output_test_work_1.2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Finished + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work2]) + assert(workflow1.is_terminated() is False) + work2_2 = works[0] + # workflow_str = json_dumps(workflow1, sort_keys=True, indent=4) + # print(workflow_str) + + assert(work2_2.primary_input_collection.name == 'output_test_work_1.2') + assert(work2_2.primary_output_collection.name == 'output_test_work_2.2') + + for work in works: + # if work.work_id == 3: + work.transforming = True + work.status = WorkStatus.Failed + + works = workflow.get_new_works() + works.sort(key=lambda x: x.work_id) + assert(works == []) + assert(workflow.is_terminated() is True) From 4b3e4bdd2abb18e3ce635d81d12f72f08497a112 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 21 Oct 2021 20:09:17 +0200 Subject: [PATCH 088/156] update requests --- main/lib/idds/core/requests.py | 5 +- main/lib/idds/orm/requests.py | 98 ++++++++++++++++++++++++++-------- 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/main/lib/idds/core/requests.py b/main/lib/idds/core/requests.py index 0a07b305..199926f1 100644 --- a/main/lib/idds/core/requests.py +++ b/main/lib/idds/core/requests.py @@ -114,7 +114,9 @@ def get_request_ids_by_workload_id(workload_id, session=None): @read_session -def get_requests(request_id=None, workload_id=None, with_detail=False, with_processing=False, with_metadata=False, to_json=False, session=None): +def get_requests(request_id=None, workload_id=None, with_detail=False, + with_request=False, with_transform=False, with_processing=False, + with_metadata=False, to_json=False, session=None): """ Get a request or raise a NoObject exception. @@ -128,6 +130,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_proc """ return orm_requests.get_requests(request_id=request_id, workload_id=workload_id, with_detail=with_detail, with_metadata=with_metadata, + with_request=with_request, with_transform=with_transform, with_processing=with_processing, to_json=to_json, session=session) diff --git a/main/lib/idds/orm/requests.py b/main/lib/idds/orm/requests.py index a9868e4e..fa62bf5e 100644 --- a/main/lib/idds/orm/requests.py +++ b/main/lib/idds/orm/requests.py @@ -206,7 +206,8 @@ def get_request(request_id, to_json=False, session=None): @read_session -def get_requests(request_id=None, workload_id=None, with_detail=False, with_metadata=False, with_processing=False, to_json=False, session=None): +def get_requests(request_id=None, workload_id=None, with_detail=False, with_metadata=False, + with_request=False, with_transform=False, with_processing=False, to_json=False, session=None): """ Get a request or raise a NoObject exception. @@ -220,7 +221,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta :returns: Request. """ try: - if not with_detail and not with_processing: + if with_request: if with_metadata: query = session.query(models.Request)\ .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle') @@ -274,17 +275,27 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta t2 = dict(zip(t.keys(), t)) rets.append(t2) return rets - elif with_processing: - subquery = session.query(models.Processing.processing_id, - models.Processing.transform_id, - models.Processing.workload_id, - models.Processing.status.label("processing_status"), - models.Processing.created_at.label("processing_created_at"), - models.Processing.updated_at.label("processing_updated_at"), - models.Processing.finished_at.label("processing_finished_at")) - subquery = subquery.subquery() + elif with_transform: + subquery1 = session.query(models.Collection.coll_id, models.Collection.transform_id, + models.Collection.scope.label("input_coll_scope"), + models.Collection.name.label("input_coll_name"), + models.Collection.status.label("input_coll_status"), + models.Collection.bytes.label("input_coll_bytes"), + models.Collection.total_files.label("input_total_files"), + models.Collection.processed_files.label("input_processed_files"), + models.Collection.processing_files.label("input_processing_files")).filter(models.Collection.relation_type == 0) + subquery1 = subquery1.subquery() - if with_metadata: + subquery2 = session.query(models.Collection.coll_id, models.Collection.transform_id, + models.Collection.scope.label("output_coll_scope"), + models.Collection.name.label("output_coll_name"), + models.Collection.status.label("output_coll_status"), + models.Collection.bytes.label("output_coll_bytes"), + models.Collection.total_files.label("output_total_files"), + models.Collection.processed_files.label("output_processed_files"), + models.Collection.processing_files.label("output_processing_files")).filter(models.Collection.relation_type == 1) + subquery2 = subquery2.subquery() + if True: query = session.query(models.Request.request_id, models.Request.scope, models.Request.name, @@ -304,18 +315,63 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta models.Request.accessed_at, models.Request.expired_at, models.Request.errors, - models.Request._request_metadata.label('request_metadata'), - models.Request._processing_metadata.label('processing_metadata'), models.Transform.transform_id, + models.Transform.transform_type, models.Transform.workload_id.label("transform_workload_id"), models.Transform.status.label("transform_status"), - subquery.c.processing_id, - subquery.c.workload_id.label("processing_workload_id"), - subquery.c.processing_status, - subquery.c.processing_created_at, - subquery.c.processing_updated_at, - subquery.c.processing_finished_at) - else: + models.Transform.created_at.label("transform_created_at"), + models.Transform.updated_at.label("transform_updated_at"), + models.Transform.finished_at.label("transform_finished_at"), + subquery1.c.input_coll_scope, subquery1.c.input_coll_name, + subquery1.c.input_coll_status, subquery1.c.input_coll_bytes, + subquery1.c.input_total_files, + subquery1.c.input_processed_files, + subquery1.c.input_processing_files, + subquery2.c.output_coll_scope, subquery2.c.output_coll_name, + subquery2.c.output_coll_status, subquery2.c.output_coll_bytes, + subquery2.c.output_total_files, + subquery2.c.output_processed_files, + subquery2.c.output_processing_files) + + if request_id: + query = query.filter(models.Request.request_id == request_id) + if workload_id: + query = query.filter(models.Request.workload_id == workload_id) + + query = query.outerjoin(models.Transform, and_(models.Request.request_id == models.Transform.request_id)) + query = query.outerjoin(subquery1, and_(subquery1.c.transform_id == models.Transform.transform_id)) + query = query.outerjoin(subquery2, and_(subquery2.c.transform_id == models.Transform.transform_id)) + query = query.order_by(asc(models.Request.request_id)) + + tmp = query.all() + rets = [] + if tmp: + for t in tmp: + # t2 = dict(t) + t2 = dict(zip(t.keys(), t)) + + if 'request_metadata' in t2 and t2['request_metadata'] and 'workflow' in t2['request_metadata']: + workflow = t2['request_metadata']['workflow'] + workflow_data = None + if 'processing_metadata' in t2 and t2['processing_metadata'] and 'workflow_data' in t2['processing_metadata']: + workflow_data = t2['processing_metadata']['workflow_data'] + if workflow is not None and workflow_data is not None: + workflow.metadata = workflow_data + t2['request_metadata']['workflow'] = workflow + + rets.append(t2) + return rets + elif with_processing: + subquery = session.query(models.Processing.processing_id, + models.Processing.transform_id, + models.Processing.workload_id, + models.Processing.status.label("processing_status"), + models.Processing.created_at.label("processing_created_at"), + models.Processing.updated_at.label("processing_updated_at"), + models.Processing.finished_at.label("processing_finished_at")) + subquery = subquery.subquery() + + if True: query = session.query(models.Request.request_id, models.Request.scope, models.Request.name, From d1650a30451a4900a02b392790d119c68b0060c7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 21 Oct 2021 20:10:01 +0200 Subject: [PATCH 089/156] update requests monitor --- main/lib/idds/rest/v1/monitor.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/rest/v1/monitor.py b/main/lib/idds/rest/v1/monitor.py index c66ab642..7036e1c3 100644 --- a/main/lib/idds/rest/v1/monitor.py +++ b/main/lib/idds/rest/v1/monitor.py @@ -34,7 +34,9 @@ def get_requests(self, request_id, workload_id, with_request=False, with_transfo if with_request: rets, ret_reqs = [], {} - reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=True, with_processing=False, with_metadata=False) + reqs = get_requests(request_id=request_id, workload_id=workload_id, + with_request=False, with_transform=True, with_processing=with_processing, + with_detail=False, with_metadata=False) for req in reqs: if req['request_id'] not in ret_reqs: ret_reqs[req['request_id']] = {'request_id': req['request_id'], @@ -80,7 +82,9 @@ def get_requests(self, request_id, workload_id, with_request=False, with_transfo return rets elif with_transform: rets = [] - reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=True, with_processing=False, with_metadata=False) + reqs = get_requests(request_id=request_id, workload_id=workload_id, + with_request=with_request, with_transform=with_transform, with_processing=with_processing, + with_detail=False, with_metadata=False) for req in reqs: ret = {'request_id': req['request_id'], 'transform_id': req['transform_id'], @@ -107,7 +111,9 @@ def get_requests(self, request_id, workload_id, with_request=False, with_transfo return rets elif with_processing: rets = [] - reqs = get_requests(request_id=request_id, workload_id=workload_id, with_detail=False, with_processing=True, with_metadata=False) + reqs = get_requests(request_id=request_id, workload_id=workload_id, + with_request=with_request, with_transform=with_transform, with_processing=with_processing, + with_detail=False, with_metadata=False) for req in reqs: ret = {'request_id': req['request_id'], 'workload_id': req['processing_workload_id'], @@ -202,7 +208,7 @@ def get(self, request_id, workload_id): workload_id = None rets = self.get_requests(request_id=request_id, workload_id=workload_id, - with_request=False, with_transform=False, + with_request=True, with_transform=False, with_processing=False) status_dict = {'Total': {}} min_time, max_time = None, None From 03bf26760e0354136f0bd32d712751f254d9bc32 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 21 Oct 2021 20:10:30 +0200 Subject: [PATCH 090/156] fix monitor setup --- monitor/setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monitor/setup.py b/monitor/setup.py index 93a35a1c..daa56190 100644 --- a/monitor/setup.py +++ b/monitor/setup.py @@ -106,9 +106,11 @@ def get_data_files(dest, src): for root, dirs, files in os.walk(src): if 'dist' in root or 'build' in root or 'egg-info' in root: continue + pass for idir in dirs: if idir == 'dist' or idir == 'build' or idir.endswith('.egg-info'): - continue + # continue + pass idir = os.path.join(root, idir) if idir.startswith("./"): idir = idir[2:] From 9b87b68c63ac505490a8c7cb3fdc1972e43e94d4 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 21 Oct 2021 20:12:59 +0200 Subject: [PATCH 091/156] update clientmanager to use workflowv2 --- client/lib/idds/client/clientmanager.py | 6 +++--- main/lib/idds/core/workprogress.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/lib/idds/client/clientmanager.py b/client/lib/idds/client/clientmanager.py index f0b58db3..78717588 100644 --- a/client/lib/idds/client/clientmanager.py +++ b/client/lib/idds/client/clientmanager.py @@ -22,9 +22,9 @@ from idds.common.constants import RequestType, RequestStatus from idds.common.utils import get_rest_host, exception_handler -# from idds.workflow.work import Work, Parameter, WorkStatus -# from idds.workflow.workflow import Condition, Workflow -from idds.workflow.work import Collection +# from idds.workflowv2.work import Work, Parameter, WorkStatus +# from idds.workflowv2.workflow import Condition, Workflow +from idds.workflowv2.work import Collection setup_logging(__name__) diff --git a/main/lib/idds/core/workprogress.py b/main/lib/idds/core/workprogress.py index f819f4f1..b70d0e08 100644 --- a/main/lib/idds/core/workprogress.py +++ b/main/lib/idds/core/workprogress.py @@ -17,7 +17,7 @@ from idds.common.constants import WorkprogressStatus, WorkprogressLocking from idds.orm.base.session import read_session, transactional_session from idds.orm import workprogress as orm_workprogress, transforms as orm_transforms -from idds.workflow.work import WorkStatus +from idds.workflowv2.work import WorkStatus def create_workprogress(request_id, workload_id, scope, name, priority=0, status=WorkprogressStatus.New, From 3dd6c35bf0215341e75245a2e1e62d6c12a530ca Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 21 Oct 2021 20:14:22 +0200 Subject: [PATCH 092/156] fix workflowv2 on processings --- workflow/lib/idds/workflowv2/work.py | 20 ++++++++++++++++---- workflow/lib/idds/workflowv2/workflow.py | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index b232540d..e97379d0 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -779,6 +779,12 @@ def load_work(self): if k in proc_metadata: proc_id = proc_metadata[k]['processing_id'] self._processings[k].processing_id = proc_id + for k in proc_metadata: + if k not in self._processings: + self._processings[k] = Processing(processing_metadata={}) + proc_id = proc_metadata[k]['processing_id'] + self._processings[k].processing_id = proc_id + self._processings[k].internal_id = k def load_metadata(self): self.load_work() @@ -1332,14 +1338,20 @@ def get_input_collections(self): """ *** Function called by Transformer agent. """ - keys = [self._primary_input_collection] + self.other_input_collections + if self._primary_input_collection: + keys = [self._primary_input_collection] + self._other_input_collections + else: + keys = self._other_input_collections return [self.collections[k] for k in keys] def get_output_collections(self): """ *** Function called by Transformer agent. """ - keys = [self._primary_output_collection] + self.other_output_collections + if self._primary_output_collection: + keys = [self._primary_output_collection] + self._other_output_collections + else: + keys = self._other_output_collections return [self.collections[k] for k in keys] def is_input_collections_closed(self): @@ -1408,7 +1420,7 @@ def get_internal_input_contents(self, coll): """ Get all input contents from iDDS collections. """ - coll = self.collections[self.primary_input_collection] + coll = self.collections[self._primary_input_collection] internal_colls = self.get_internal_collection(coll) internal_coll_ids = [coll.coll_id for coll in internal_colls] if internal_coll_ids: @@ -1497,7 +1509,7 @@ def get_new_input_output_maps(self, mapped_input_output_maps={}): for ip in new_inputs: self.num_mapped_inputs += 1 out_ip = copy.deepcopy(ip) - out_ip['coll_id'] = self.collections[self.output_collections[0]]['coll_id'] + out_ip['coll_id'] = self.collections[self._primary_output_collection]['coll_id'] new_input_output_maps[next_key] = {'inputs': [ip], 'outputs': [out_ip], 'inputs_dependency': [], diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 081de954..3c90ab0d 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -1650,6 +1650,22 @@ def last_updated_at(self, value): if self.runs: self.runs[str(self.num_run)].last_updated_at = value + @property + def name(self): + return self.template.name + + @name.setter + def name(self, value): + self.template.name = value + + @property + def lifetime(self): + return self.template.lifetime + + @lifetime.setter + def lifetime(self, value): + self.template.lifetime = value + @property def num_run(self): if self.parent_num_run: @@ -1750,6 +1766,11 @@ def clean_works(self): if self.runs: self.runs[str(self.num_run)].clean_works() + def is_to_expire(self, expired_at=None, pending_time=None, request_id=None): + if self.runs: + return self.runs[str(self.num_run)].is_to_expire(expired_at=expired_at, pending_time=pending_time, request_id=request_id) + return False + def is_terminated(self): if self.runs: if self.runs[str(self.num_run)].is_terminated(): From ff23e67ceee614481d0d12bb7f0dd9445766a2d0 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:38:40 +0200 Subject: [PATCH 093/156] add api to show work relationship --- main/lib/idds/rest/v1/controller.py | 2 +- main/lib/idds/rest/v1/monitor.py | 44 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/main/lib/idds/rest/v1/controller.py b/main/lib/idds/rest/v1/controller.py index 1f8eaf97..c418a5ea 100644 --- a/main/lib/idds/rest/v1/controller.py +++ b/main/lib/idds/rest/v1/controller.py @@ -50,7 +50,7 @@ def generate_message(self, exc_cls=None, exc_msg=None): return json_dumps(message) def generate_http_response(self, status_code, data=None, exc_cls=None, exc_msg=None): - resp = Response(response=json_dumps(data) if data is not None else data, status=status_code, content_type='application/json') + resp = Response(response=json_dumps(data, sort_keys=True, indent=4) if data is not None else data, status=status_code, content_type='application/json') if exc_cls: resp.headers['ExceptionClass'] = exc_cls resp.headers['ExceptionMessage'] = self.generate_message(exc_cls, exc_msg) diff --git a/main/lib/idds/rest/v1/monitor.py b/main/lib/idds/rest/v1/monitor.py index 7036e1c3..26016450 100644 --- a/main/lib/idds/rest/v1/monitor.py +++ b/main/lib/idds/rest/v1/monitor.py @@ -461,6 +461,47 @@ def get(self, request_id, workload_id): return self.generate_http_response(HTTP_STATUS_CODE.OK, data=ret_status) +class MonitorRequestRelation(Monitor): + """ Monitor Request """ + + def get(self, request_id, workload_id): + """ Get details about a specific Request with given id. + HTTP Success: + 200 OK + HTTP Error: + 404 Not Found + 500 InternalError + :returns: dictionary of an request. + """ + + try: + if request_id == 'null': + request_id = None + if workload_id == 'null': + workload_id = None + + reqs = get_requests(request_id=request_id, workload_id=workload_id, + with_request=True, with_transform=False, with_processing=False, + with_detail=False, with_metadata=True) + + for req in reqs: + req['relation_map'] = [] + workflow = req['request_metadata']['workflow'] + if hasattr(workflow, 'get_relation_map'): + req['relation_map'] = workflow.get_relation_map() + # return reqs + except exceptions.NoObject as error: + return self.generate_http_response(HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) + except exceptions.IDDSException as error: + return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) + except Exception as error: + print(error) + print(format_exc()) + return self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error) + + return self.generate_http_response(HTTP_STATUS_CODE.OK, data=reqs) + + """---------------------- Web service url maps ----------------------""" @@ -481,4 +522,7 @@ def get_blueprint(): monitor_processing_view = MonitorProcessing.as_view('monitor_processing') bp.add_url_rule('/monitor_processing//', view_func=monitor_processing_view, methods=['get', ]) + monitor_relation_view = MonitorRequestRelation.as_view('monitor_request_relation') + bp.add_url_rule('/monitor_request_relation//', view_func=monitor_relation_view, methods=['get', ]) + return bp From 0ac40932e4999809e3b0da9473eb2f8a8b759605 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:39:59 +0200 Subject: [PATCH 094/156] add function to get relationmap --- workflow/lib/idds/workflowv2/work.py | 10 +++- workflow/lib/idds/workflowv2/workflow.py | 67 +++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index e97379d0..494e5f56 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -592,6 +592,14 @@ def workload_id(self): def workload_id(self, value): self.add_metadata_item('workload_id', value) + @property + def external_id(self): + return self.get_metadata_item('external_id', None) + + @external_id.setter + def external_id(self, value): + self.add_metadata_item('external_id', value) + def get_workload_id(self): return self.workload_id @@ -714,7 +722,7 @@ def errors(self, value): @property def next_works(self): - return self.get_metadata_item('next_works', 0) + return self.get_metadata_item('next_works', []) @next_works.setter def next_works(self, value): diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 3c90ab0d..5f96ab74 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -710,6 +710,7 @@ def works(self, value): work_metadata[k] = {'type': 'work', 'work_id': work.work_id, 'workload_id': work.workload_id, + 'external_id': work.external_id, 'status': work.status.value if work.status else work.status, 'substatus': work.substatus.value if work.substatus else work.substatus, 'transforming': work.transforming} @@ -728,6 +729,7 @@ def refresh_works(self): work_metadata[k] = {'type': 'work', 'work_id': work.work_id, 'workload_id': work.workload_id, + 'external_id': work.external_id, 'status': work.status.value if work.status else work.status, 'substatus': work.substatus.value if work.substatus else work.substatus, 'transforming': work.transforming} @@ -741,7 +743,8 @@ def load_works(self): if k in work_metadata: if work_metadata[k]['type'] == 'work': self._works[k].work_id = work_metadata[k]['work_id'] - self._works[k].workload_id = work_metadata[k]['workload_id'] + self._works[k].workload_id = work_metadata[k]['workload_id'] if 'workload_id' in work_metadata[k] else None + self._works[k].external_id = work_metadata[k]['external_id'] if 'external_id' in work_metadata[k] else None self._works[k].transforming = work_metadata[k]['transforming'] self._works[k].status = WorkStatus(work_metadata[k]['status']) if work_metadata[k]['status'] else work_metadata[k]['status'] self._works[k].substatus = WorkStatus(work_metadata[k]['substatus']) if work_metadata[k]['substatus'] else work_metadata[k]['substatus'] @@ -899,6 +902,14 @@ def last_work(self): def last_work(self, value): self.add_metadata_item('last_work', value) + @property + def init_works(self): + return self.get_metadata_item('init_works', []) + + @init_works.setter + def init_works(self, value): + self.add_metadata_item('init_works', value) + @property def to_update_transforms(self): return self.get_metadata_item('to_update_transforms', {}) @@ -1324,8 +1335,11 @@ def first_initialize(self): tostart_works = list(self.get_works().keys()) tostart_works = [tostart_works[0]] + init_works = [] for work_id in tostart_works: self.get_new_work_to_run(work_id) + init_works.append(work_id) + self.init_works = init_works def sync_works(self): self.first_initialize() @@ -1418,6 +1432,30 @@ def resume_works(self): else: work.resume_work() + def get_relation_data(self, work): + ret = {'work': {'workload_id': work.workload_id, + 'external_id': work.external_id}} + next_works = work.next_works + if next_works: + next_works_data = [] + for next_id in next_works: + next_work = self.works[next_id] + if is_instance(next_work, Workflow): + next_work_data = next_work.get_relation_map() + else: + next_work_data = self.get_relation_data(next_work) + next_works_data.append(next_work_data) + ret['next_works'] = next_works_data + return ret + + def get_relation_map(self): + ret = [] + init_works = self.init_works + for internal_id in init_works: + work_data = self.get_relation_data(self.works[internal_id]) + ret.append(work_data) + return ret + def clean_works(self): self.num_subfinished_works = 0 self.num_finished_works = 0 @@ -1658,6 +1696,22 @@ def name(self): def name(self, value): self.template.name = value + @property + def username(self): + return self.template.username + + @username.setter + def username(self, value): + self.template.username = value + + @property + def userdn(self): + return self.template.userdn + + @userdn.setter + def userdn(self, value): + self.template.userdn = value + @property def lifetime(self): return self.template.lifetime @@ -1864,6 +1918,17 @@ def sync_works(self): p_metadata = self.runs[str(self.num_run - 1)].get_metadata_item('parameter_links') self.runs[str(self.num_run)].add_metadata_item('parameter_links', p_metadata) + def get_relation_map(self): + if not self.runs: + return [] + if self.template.has_loop_condition(): + rets = {} + for run in self.runs: + rets[run] = self.runs[run].get_relation_map() + return [rets] + else: + return self.runs[str(self.num_run)].get_relation_map() + class SubWorkflow(Workflow): def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None, logger=None): From 87c783ef5ebabc11e00d662d625e024d06cc844b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:40:47 +0200 Subject: [PATCH 095/156] fix get_request core --- main/lib/idds/orm/requests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/lib/idds/orm/requests.py b/main/lib/idds/orm/requests.py index fa62bf5e..9c801ed1 100644 --- a/main/lib/idds/orm/requests.py +++ b/main/lib/idds/orm/requests.py @@ -221,7 +221,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta :returns: Request. """ try: - if with_request: + if with_request or not (with_transform or with_processing or with_detail or with_metadata): if with_metadata: query = session.query(models.Request)\ .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_SCOPE_NAME_IDX)", 'oracle') @@ -428,7 +428,7 @@ def get_requests(request_id=None, workload_id=None, with_detail=False, with_meta rets.append(t2) return rets - elif with_detail: + elif with_detail or with_metadata: subquery1 = session.query(models.Collection.coll_id, models.Collection.transform_id, models.Collection.scope.label("input_coll_scope"), models.Collection.name.label("input_coll_name"), From efa1cad794f0d1dc3cb3775c2c2666b55b6de40a Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:41:48 +0200 Subject: [PATCH 096/156] add subworkflow loopworkflow docs and examples --- docs/source/general/v2/workflow.rst | 18 ++++ docs/source/images/v2/loopworkflow.jpg | Bin 0 -> 31296 bytes .../source/images/v2/workflow_subworkflow.jpg | Bin 0 -> 31227 bytes docs/source/index.rst | 1 + docs/source/users/workflow_examples.rst | 96 ++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 docs/source/images/v2/loopworkflow.jpg create mode 100644 docs/source/images/v2/workflow_subworkflow.jpg create mode 100644 docs/source/users/workflow_examples.rst diff --git a/docs/source/general/v2/workflow.rst b/docs/source/general/v2/workflow.rst index 6208ff1b..6d81a46f 100644 --- a/docs/source/general/v2/workflow.rst +++ b/docs/source/general/v2/workflow.rst @@ -21,9 +21,27 @@ converted back to Workflow or Work instance, which simplies the handings in the Workflow ~~~~~~~~ +.. image:: ../../images/v2/workflow_subworkflow.jpg + :width: 45% + :alt: Sub Workflow +.. image:: ../../images/v2/loopworkflow.jpg + :width: 45% + :alt: Loop Workflow + A Workflow is designed to manage multiple Works(Transformations). It's a sub-class of DictClass. With Condition supports, the Workflow can support DAG management. +SubWorkflow +~~~~~~~~~~~~~~~ + +A workflow can be added as a subworkflow of another workflow. In this case, it can be regarded as a Work. +However, this work will generate new Works. + +LoopWorkflow +~~~~~~~~~~~~~~~ + +When adding a loop condition to a workflow, the workflow can be looped. + Work ~~~~ diff --git a/docs/source/images/v2/loopworkflow.jpg b/docs/source/images/v2/loopworkflow.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4cc02e5f364314af97aa7da465f0820957f65bea GIT binary patch literal 31296 zcmb??cRXBQxAz${dhfj?B#0Kh6GRW9cN0PM-Wet&x)2g1YJ#Z2FM34pLbT`-U6c?* zg26EFkzaY9``r7-`@WyMWuNn%b=F>ctzFkH!Omcp017QNO*H@l0f49AAAns3{8WRS z9{_-k4sZhi00IC8VFGYL1hN1M5axfd8iWUc{-)yqK#Vg0`-?{xd|w7o+V3=fzHwgR z{LKMLe+B&;1DH!z>@Fa6*U8h*)7QziJ$0RZkEem(~3${c3q799BN03K)! z1;7G`+S>VgDc!w$=The1@89zOc^pms)*bjPa4G8_^8X#6vUl*c1C6%=+3(x?+PQZ7a-;V`Mp5g4q~bM z|0dJ%->|Li{eR@!+B*J&|FjEC2`X{h+1Ja>Hu(3C|ILTHn?G1~zr%I#N#f+AVF12q zz%rb1_0+z^Bp}}LxUZ`RVk!`$ok8pV#5hDw{u;(02KjMV?0i+gas}u>eAC|cwl;`a zK&)@+u_!rMBA5Vk7^1`Wq;G?4S7ro!s z2Sk77wczh+d`Z8Qf8gYx`ImesS%{O7DM%**04P_0v&m)eAOo?WudCsuKA>(;xezCn zOZsmfAAf^O{kcH8frE#-GKfK2paBnj)Glqm9Od z3rq_m@^bs#Pe3~Cs)L*QCH*Rh#e4${|4Iup@$)gdq=S0G9(eiPy(~jePgsbHt=3%- zgZjXp0(Srv;5NVk_=E92;0d?@&cizmJK(QBzo-GWfDhmZH~<2FQT|S0`umFq_%sAU zfL*`?qW|<@f)Y{x|hsoJyd*cm7K2|96fYn|Ev%i-S0UB zLih1sv`1hcm-oP8vCaRYX`cc>?G>>5Fa3+gJq`{N;5gl%V(0G@@TWZ3)2fhO+;0QQ_Kp^-KQV2DK5yB3+2H}H zhylbLau4DN@qqY45RgZZ1V}0*6Y?5T3V8>459xpmK*k`mkY&ggZ}Hpk$MM(k z(FDW<>;xhNY6KPp9t4pD=>%m2tpwu)8w6*B6ol6ZWeD{M?FoYklL-q6n+Qh;*9p&v zD2cd<j_`bkVoe3e+5SfALL_#yEN;y;M{iI<5_NT^6|kf@MY zk@%A&krb1(lgyI*Bqb%iPI`;fg4B;RiL{ioi*$h$O-4;7NTx|vWoQZgk0X)pS#Ir}V7!O7sr&arBk+yHRcG~LeaYIzy2D1tc8kr0?KxX3+ZH=Dy8^p2`*Zd-_H7PY4kZpZju#x=9H=WS zS2V8pUwM6H_{!N;?yJUEBd=Cn{mMziDaHAK^EqcHCyI-mONR@=Rn9ecjqsY(HK%JC z*ZQs8Kc9bEfJ8t^AV8p8U|En(@Qz@lV1wWfAub^cp%kG$ zp^Ka1H{EU)-kcYv64nxq6mAke5V;{@Cz2&HAxbK$CW;ho6x|o&6LS#D6`K*K7S|Dv z5$_Z~myne3k*Ji|lH`)SCz&ldEkz@BS1M7eUm90hSsE$bEPWy)A>$`gEwd*pAnPhy zCc7!eE$1LtD7PwqRo+(qwfvF-hr&IDR|-ptR}^g(^A%Taao)1ORdj1ZiC4)*>8;Wa z<(tYr%5}=eDl#fzD($zSx7BaQ-yT+_QZ-i1R9#TJs^+Lxu7*+5`d{S(w?NIg`1wd7}lsg^5Lx#i6CDWt!!>mAF;3)s!{2b&&Od4U>(VO$(eHZVRuu zhkMWTUfDg2t)6YZ?csgR``PzVb}DumcHix9**~}6a*%gOb=Y_y{~-0jrlY*$Gsi6_ zMW=M99cLBiOy_+U4VOHZV^=-bVppu2saushf%`r8_a0Opt{&ZMEJ>zC)lJ1 zNyEt!$+=JQo_ai;N>NNHNu^8;NnL%W^Q`_k$Mg8-ztZ4o1Lf?~T-((jum!grf6ekK&~ggOcu2$oO1v2t+!ThM=NeuG*${!zOJIH zO85i#!~c)%clX{+Rclvw)=1S<)?Ta4uA{DtuZPqJ*P|Mo8kQQ(8^_*jz3*(2ZL0eq z_@TJ@N^@2VZA)@1QS0MYY+Fd%VY^rRPKRU1YA3vNzRRp@qFcXvxJSFEzgMldt52z~ ztzW*sc|d01{YR;f4TBPc^+V!Ab)UpP)eTDw*N;e!G>%G-HjT-RwT>%}cYMD6xpzWy zVsKJ-a%{?YYG&GMdTGXfW^>kUc7HBl?ri?y7u+xLUn#z(FR(1+FY+!{ElDi3EZ<%p zS}|OiU$tBPzUI4zS%3VE^jrD{`$p-e$Y%4F+Scf{<@Wk_ukU9&(Yutp**|!G)b8Ed z8$y|(*7tq(u?LAi8Gjc468qJ8c=vGO$o=T-I3CT2E;*4p={q$&-8c)x;A38#^PV?d zXkUE6x?{0kwm!DMeSyn$H_iiam3~kR0Ho#sKs^fj50-zf1%Gou{#?I;81h%VB>xBg zbG>-^0rVk&O7L~*#;ddgz#m||{t%Q6#!DC8=oWw@D*8_ZDVM?bfvCuD03q=KT@y4G z`;!#_@S6bO+y{$2%fVvL3&HioEC957|G5UeU z|M>N!lTx=#lPxJBNCDFnt)UJT?ehf7>Y~u zzE30}5gk1PBh$6(+&sKD#Ka{erKDw4ZmX)PYiMd28X23Ig07sMy~6`XCubL5KmUNh zpx}^4kE5bvV&mddpFK}Y&v@}NGryqlO;K@4X<2nmZC!ms19S=ayGi*VeynY;JAuAN>4vcyx?DIlZ(C0zm&X>mSSh!!AnDE*uyP3d6s&3xX2} zMkpl=_lhtcm6ATbtvB^mkq81B<&@X&ItV#M4N$cAea49BxWtyP?O&Sq+p_;V!y^B` zEc?f>f7vw;y4^Ux0~7}biVKB8aq)1$fJXp&X?S=9L3Sx5~$M_yhbtI{sW!Su}zykbAq~|N|8Bu!=+ZyOzy9bg##sW@+PtRY+KC+1y zOXBEznxU^&QX)%8>;2iQ*UaxNU(=&h!;q521=l9{=P;qt9=Xovj{S=#?8U!~OpYn6 zJ6I>*CWwjh`1xH)n~YZBWuZ{#h);i?H2PmO`%g{(-+x1~qyKL4PrY~~y&NIRx65u^hSQgFv|>L(3S zrb9f#7{yL%pkV~gr06ulbTKNEJr`aqc_b9=Zf+q@wh-36K~fbmSld5q-B0weCE7>t z>5Vd;+YsX*zze7%ms-&>ll|N^=O4PqaO(FA3k;YZHw8&GcN!ks=40R|6c%$TPEq$^ z8L~Mq)^n0g?gw*CjH|PhjA^|dizbkZK;Q=6fS?Rj&0 zy^A=4w&V6xt{23C=ev*IIaknRXVl&g>f`DSH#y?=I!{9T@AGvaX`978S4N;N`r8ts zULCsEqILH_F`lIKy=FbET>g#KwvY4Fv&^R6W4-8~!f-nVa?Y zUhon;g8Jlcsgnmr=@)-xx^|N8&D$qfz>7TXyT)g$%EGxslii)Rr6X9Uba;ds7iIwZ=j)9pg%R|p}_~=v38P;=V4m7z^ z!(4h0g26pWg*NQ?K^>cma~UT>*V;^u;PSEEy5-IFJ3-_R71tUNcTSy=rySG9t8K9s zNf8xW;bf@Fbn#>gmyZX9dfaIwH?RBhCqL>{0>~QWHh`1I7C$PMgc4dMCo7xzDz>LW z`etBMTT?o6jsp+z2B`{4`(OuTF7APMrIKf*g(Vbn^7FberEvpeF<(hjywN zB&nmhIq{i;x$I5T2!;%KOumDY+C6W>zq`so?k}jdd z!o2>~GbP^41y`q^EzeULX*;A8Vv3}{nXYVfhwhPoTHWRBq|{uFee@C5VCl)#kcb(7 z3~m|Vc#qVpbBkCL+n`%kSyj4|-&i{_-DVP)%5a(9f8m(vTI(;gzVw-FA|mHa z(1%_o#V$kCM^9Xrkrf7auFnPb4WFLj;fX&V$@y?gn)|gp%16x1#X=mm{$}!*uJU9Q{(MbVG`{!qxk$FfS@pY!EK+K+VQ(b+vqPM5~MrYsK^+X=Nkcj3Bux{Vfu;jppntKj_mHv+HO^ zd%T%dWboCt$+hOMuEf7EDlxJ8^}*l9J4{|8ys1*g_>HdBsN=6EEO9sZ-`(`{BKkub z#`c(UE)O1;2WfLbGGNG^5Q-~T6yxM5nAy}+tY`RG(in#l%TNt&2zICL|IyIjH>Fs zR&DyL2vzVM^n74<(1(vWV#L*Z$xr3anXv#Nt0)#o9Qz+~YdflBKN-3xH+*+aI0nx_ z*5L1|7q)ekKilVC$;X&XB5NPamzz1?oVoq<#M`|=*s|=3z3hv65tWA_L??W!^t*bi z>*$=7E6YN2^LJ1tNrG8BbK)b>{DKeOB||CY#=D2t*nGNkU2}7hkdJny{!Pb%vpro&pd9MZ8$30m&NElM5vq>^M`Y9VgZN7 zuLb5o0mHqH&!G1CPMoGV3WbjNEv|;rot=UPC8@X7@r^(5^{dxmoOF8h6z=4w~fWC_olP!i0<9qz{Fp)`x*bOm?p-CQE8kby7{h66Ta<} z3AF_ekjj4l3zbz=kBlrDFE!@d4Rk%FC26g#8l>9^OH)P4=b%UwWRq{LDiL0%n4GZ^7#s7r&DCAB!$PXbv&Ly!f#Hw$nZ0r(w|{C= z%RUVrqzbMK3BDjQ^jkHS7MMJQb80#qrW$w$DEc^6ZPZ*a6|CZ)vt2yF0*{kkVu4=Q z1uXDjaW1QSZ(jxiw%hD?=)8jS^woeT4$JehjJbD6Q{!Geu_8E4$!0sOxU)e_@Lnrl zlahtiWG6p-NFSb|SeLx3Veq+25mnwQ*?|fh{CZw`NJ_V;PrphT>=L;ScTD?9-N3jY zI%?Vr|Bq5KRF~~EN|u+2tNgA%0yPmFfAaFO5&sV@e|lT5TX8O-eRVm!bq+t_8fxL) zd4}R_UFlmoC$~zug8HpxL_wM4uyF*8gfc$!-}{u2$mw_+zva6lV_M7+vdai@4+|75 zjnbn*Zhl?$K?XYz?b9ee_W!2Qj;=0Uwv*ttgjfk~Ft|gNZiLPsbqT-KB zXD%W)tA@#EJXaKV30H*T;mqMoh`{|~3^%mB+KI4eq4K3`ky_W?=xyhh26H@v_Wbu^ z1R|U5xD^WyH?L%`7uS#O~EoCG>2~V>#QvJ&Q&y_O_6*|m+)vT7eB3xtEPU96ryoBd!<5N80UO-Xe z^DuqlBY14C=>A4xxIoaO3u?VsrsAWDt&!Okz4Y;MQ4HO_pMLwmS$E<)fm9R07JdQp ziu;~}p0w$*+?2CwVK3bmVY<406Y$)IpsURY$#McN!=JHcB;rYY2Do(!87ndj0f7P z(wN3_{Pyju#jQit$GEB!Og?BkPB?12dSQWsac6~Xq4(uaUFhgd>FLrNgmvUgah%2` zb49v1r0{QOY-LueyUr`kW)BxHwI~|xCncXo~zA=rkCW$ z4?V5yTJGykW<@aX(_^@>0J#w287DN*R?uR2U)`C4GJ0(}gG83?{px0w9^a?$OWUDZ zbraG$#dH6i!8rZ@v^bNY&d6~&UA>;?%C$KjmRykzKiiB#aNQW z3+eFj#TxS7FwtdrZ`k#ps!s%)!}e7S_VJSPe|$MF(9MJkU*pn@9k2 zW4H^O#U2AO%X{4VlDT#XoZXHyM4xcbm!=9tt71zGpI6vP+S^}Eo^STPzdw{FQ=qP5 z3W~Acs~UI#M-{bFAV;6PMlMxPV}U2rp~1z@r%xWwnY?zTolNU~z{`aQu5!{6bP4!&~*3{THVb z9L-+`&XWF1imYcOdPDfduJOy&K+e19VkSE*nZM zB5(ysv+yQ5%l>s?+JX18hX5o05O_wcDbM$%`*WXR6k%C)_@p)0*W2 zA2S3`1*iF)Y?x7dQE#(!P{SFng%+YONhG|~JsAT4Iz9&d#)lB85uEqC+RL6@0uN5z z4NXze$zh>B^Xs``3Jd1;o+bgx-Dl5igC(BQq~K_uGaj!-)6YTmWMphDU;bl+I1V^~CkC8fVK zNiVart;XnEnRs}EYuhWi2ehBR`B;^Y{y;A4AULqVYi4CE(BV*p1(++ssFz2&7D?fx zhF!S~^c{rZ)`7^Joy0NW_e>=DsM-rJXA7l6%h1TDX({1suQFitQ6?->w z_Ijy9r8m`;U5Pc%D_V)Z+Xy6d(ph9LxOe9hROaGon{(MkK?c4+WWw)K4W>p9HkvY3(#bqg(yMX)Zj)4e5JX`$+~ur%ns66#LdpV%tt_LH`L11j}B z!fNC{t9MI$AKBd&%O-*1-c?y@5wW)LMBm0}vfRy5i6GCp66!lIO*Yo*yU5Cycda14 zTg%AF*t^z&t8#Lua=gLBIWgykv#xi`oMqtXDfKWvvi~&H@YUH|=?f?JDa3 zk)<^uaKEX#F|mh>D!*BenS{pvo>3_ag_yCdlHMtMOrQDik@L@r#Ftm9$#J9{ z6qDu4XUe;zw$2DAOAA+di{_6FOap(`)&EG6`sPP3lNjJ4FYZ2boQF1q6DV$~jkcl8 zUBQ_Z^mwAgQI9SjyAxvp-1Ih#N26!#V(nsQUbM_A+SR%__q*X^-b`XC#43SpV{oEY ziIjA&CNx!=_C9=0Z&?eY01C*!#Wct$e|eJ~DP=Z7^FtCAG)%DMC{M7d6Z{=Nt5G@M zSlYo=G1PL`(l|2bBV0w9KmUgT1OEEx22>7ksW~Feu$7eWhujCkAE4BQfQ}+zJr?HJq?t(-9(%1s=6B*;rZGpr4uiQLSJ^ zDRyC=e(g%}y>J_m4e2Z4b{%+4H`W-Vn6xSPcn3ayHyV%E-TOFFnk|eD9{Bt@&0?ge zqFktPx9$I<7o0p7KsTHzX*3;x^sEdcc#y2?>uTs`$Yk+9@n-1zb(5_mtxVRo#9!l|Jz<)@EMShg0auyj+j_ z`}Sd(B~#FCCvsiJKPQ-@zaSPsqQ3_HC^&n7Ap3k6I39KWb;7%~_e11t?hJ+Z(^-

rr0Su=F-Ow5ovzefvvJCv8o194T@S& zOi?GgEgHS8^TZ4v#&u3)^w1q`g6;1=2Vm>nZ#l)=64cHXQEyF6OetR!MjDyMt=dK= zCK9D_MiTV$eM0c=XSKJA?)Sb&{Zx6Ms$ znSVXF6@mr+qnm6$S&RuMMu;{3F@1gX7o6P6+UiYCMYqXK6;7)=WHV!{w>n<%?#;E> zv>KhiZCkTJ(Vr(V!7n1@{TxIq61vsN4c=A=9EK(CyU>(2rNlQKD83qr#sX9b!j&yg z4DE7-3#p0~DyrpiNK4`Em;4LcRH!%z1MVb4qT}l-!>YlynMW78eMjY3pmKo>3yd>2 zo!_qdj*vvHZYB?uVeatc$IlFb58&^J{d|tjWT?CvBuHr=uX*&BbXm#l}F|%-R~PRD)xZE;zQE=vv;OMLR7l zw&yO&l0^!ugYS00f`?Ki2s=0);ulv!&Qa&DkPdKsgmYZou6T6BD#1fx7h$16iLnUQ zaq`2sq?kQHJTj5ulMD3fQgU1yPOHpT$leo#aG-Fq-ZR}C{)}BV`*1sr3_7$_irF8- zH%(y@Q!A}{jGdGQX$bBXSk`)IG1^AK)#m6cI+^%}Z|$MAENL4$YlWQv4X%JK_m3iu zbd=|o1>9fNz0k_2))L7zw7Vg*Mxg;5%9$ADQDcIa-BmOdDVrrdi?=L>w%ju48Wemj zwYk5TKTl*k;p>HbPh^3!X|J@!vk^1(#w~yNK-jEUW(l92Ux=VNF2w;G(N`q-;u2LL zq>r0cuV=2cW&gQTF7y4V)=xvxY>?@sHka#f{(y9%|5oT(Yx%Y#c5z1kIIF9yz;}QU;Qs9K! zSm@x^?S6Wu#Gg`K*m2dXhv6Xuj+%G2=@Xr$Wcsf`)ZNd_oPBdt?(9E5?E(9Ct-CLY z?JePGs24{n!SD`+NKjc6iQJq1aZMlBupf1;ZK$q9-i8Ms=;Z=e$YUihxX{SrpvA5+ zXWP0n#^u}vqzt&AzmNRU_=Vhxg~hK*4|P<*8C1R^#zr;jZfN!{cZlBE9Cx)zc&HdA zIQD^AU3z1xQI9_8S$7+y_y|n1C`iTR-RIg*9v?$Xs|1GJlGvY8*q=B->7OnuwM&gC zB%r=)P3*LCA=Fx%Ycq*m=d-RynsYj6M&4eI7Y1&^5?7^)W2+1HRIjHGZ0!pvbd`GO z7}w5!OCnz~dB6HZuYH^~e4KBW+F?sl)T%jyO3LoVECYK+$#96N(sEwxDgEOM{4E>7 z`mw2T-oY-{VoOnplZ9ZKt2JwwZaS{D!(LrTKU&)N?WPUWgaYGtCcFaoFvp_}dI68{ zfvfMX1q|kzCm>!yv;8Z>gQFw$$3QS= zktS`?)f`^q)SOgbYd@FqX=LB44Tkr!yZ)!S5!~HafGx=K?zmpJC+YmwsY#8D<+YxQ zk?g7Qfnz!cPW)D_{f8l|xPsw4=so@Ym{>W+r9Dzrq(Z_~sISC+ve44GpI6I zoKU83&DQ1L7a7&!mgV_6Pm5AKS%1-oL=6{s9tABYv8+=(`}ux^p<}C$D`{hSDy?)D zp@Du97VIoWH{Bqi1FKg1_zV`;MROmxN66UgZS}KQ>!xouy@(}TFYelO-RaX+TDm#Y z-Jk0EsF6PP75o@>^ zT#C-9m~Ds?gYSGOo)kw?+hj2-5yw^DUa;uBZy@qewGX0+R^d;8#t8{+rP-b;{M_M1_0dcW1AK6QRG` zxfdd!=dWu854<#=2M7o4tXUJLx5K{OPHfHhTpvvaK z7cO3j)7kF_j*HJtV=*%5jW-DO>B3R|W}&Lhadqg(hD(69CxqbR2bDPXZ?e^RGEj>) zY6ZcqMUrcuSK3e)H5ECxwT!kT5_~uh>en=zZqMoNWmI~(o)DQUTr)LuBoF)b88b3F zC6X0W+Zr>lSf~502pzh*(U}u!_QKN4`Jl^Nu8@w9CeUek=B{Qt3`JO`SQpcsM~E>) ztJPqnCe_}#zjmpw-d2`+ll)P0C0EKC(|xBDnUQ#)UKb)6tDu13-*@fej&G$z$C!uS zyRCM7tG-bxz-H8)47RMBQ0OE}f$yRXi3_?@JocODrL(|)u>htDx==s$(M=7LOk z$VqrV`wlNyN^`l{Ig!q!`-Zh;^Pp~MzRfx}*KydyE4!@6$g1WbaIP;pd~87)?Ypnt znMdAy1@$`h%<}BEraO$XTGN%fmy?1eqBdi`{o+T+y_~(r3Sy|h&b?$fZC{V;^UvbM z^Y1O@R+br}ql#12j5&mHjDlv>y?|e|M`?>ut;`7C&*JRNn>J+i@hcy%r$ndOkfqR%gjH=x|HvWuet4ZdEu21o^%T zMKZBJ6t5!P%9Z?#VaQ}_AklW`pH8Q=w0UW(#FC05G7}!|7THx1`aU}JFywCE3a|Jz zX`dcV-^A*eyQ2d6(e_hfkOeLi^0i$y5>b7#-So z>eRCrEzWhN{Ve5+-8am&B*oztzHWGJZ_M(i(7U7DD0sXl*XQC?yv%uBO^&C%J=$vD zY~PcKkm0GjtcxS(96Q)JJ{OIPFGlCym~is|z$1Xo(NEO7SEAQ~&9uu-I4@>PFIR1?;*ne>Lo`C%*S4ppsM4!&u!tj_cmJ_gw4TzG9!yqYM5t zAq*Uq(9QI={`#exB@zzgfR`_lK#)vKa+%O^RC-~n z%*^IupKPqs9bT&RF{N`83!T;^CB-)V-lh&Zh+$}TGWY-$kT`oq<@M&0H{4ueS)(kVUuYem&Y7%4kciSYb= zvtP9MT$8~~1_u0()oY>CdFb#yn9l%ugoibQ1B<#5<)gN+kR#tfT$*ub;oQcen#h!ar`xtF?&N;1o{xI!m zD{Z@zRaWtd`*;yE!h~|{UVJ>~xbvhUh~6-lcAw>Yz!RsjXZ+J5{T2)NydxMIKvP3Y^$w3IaA8Kw9EzVmmU?XBj;j~njybnWRt(hBKAj)!q5>b z>k4#e^)mFc?wX8jf%GWH;GlJd2hZ$%YTntr*-x1kZ`+3+m7`U0C;K85crl98`As!W z3+|HU;}z)|IZx%>P*%E(n(8ElC9;;Bk8JW9dU|V~$g#|2XK&Db4tQ>5VPo+FlVqbM zNT24Aub-g#EAYe5QS>vtrhU^+S8M!L71;sFWVgEh#rj#RX046+P<0;sN~x-G_^fQl zA}R($G1^^Py?^KvbuXYfd z31v|~=7>PL@8kPEkI7%^y?KcX%;(t~V}71Yn@kl=+g|-s3GS{;YVYRLlwav^nawxe z&*>b zsO~|KGs|zVz$W!HJ9v7w1#ZYZJl(s1&VifGI(qwjtK^RmhD)F5dvayauB+vG#IrUQ zE>jj+TlW_R)*(?J5~5$IZLa8LkVg0S1W7BBK47r2mZ>Yzms9X!LFA8N`^JWsqEE&K z7#(b%sws(|ODhW)#)qYxq^So>xazCYmO@l%?*1L|u@isZT&R*mzeLq^u+t;Ng`yDt z#n%oCSjQSq92=h1J{_23{^HL

(x{EWXMA8X>RhY`z2SwFk9|MB!2w!wWKK!l2{n z(8E3a!7FCs8lxEvFidGc`7yZ^`LWrr4q5dWuDBb7ocBI{i6m31+B`b?IATzfr8jX! zJHhp_(^`@$NmAlXMjhE9EHr)AXNW_!>jzM2lo&TK@v$p7C)osmo+PjaQ z>BoP#r)jO;wFjW*SB~Ri9xJe#6AwP;J1T!KQlGB>juSBsc#kftng5|2sm3=QjuSo< zE`r|UXF4hzM+dflXm$O9`=QmCOJt4@2k4)(x2+8zQ{g#JMA3Kc;bVkRlWmU?ev=3|PdIo+T!V~l6-Eb)sGyx!nf0Paj<$sA7)_OGCo}Ye>48C( z5vo1XpTRM{)Ns@3mk48Y+mhlV8)hUGdgCtLSaz(yGgoVCsdir2?Z`C`#?3M^vxrZ3 zc$%Il-`M>nXsB=XY33whX=7)1UvFt7wvL7||VgsMG6s z)eIyH=x4xyKu0;#K6q`$F3}<-#?rM2L)((4gsOi@`l;pX(9IrTVlC}Cexln?V>@G6 z29aki)df&Fz23IuTm|%5$?kN5(*kWo*WBhGtwt&1+QpDNRbQY7Rj3l#grL*1{KvO& zO4^|hfvTKg(IE-4U#PLf?Z9g9M!&RqXQLcdMcmYcl#g<1GZL5)h1M))E!P>4y9& z9Y?(UZqiOgD*jCpyZy(E4BV?Bo6VG~o{R{=eg1GkbXZA!4x%s^m8w(s#qO(ZSau)5Q0oO%iHH(Zf+GsU9Vei z!-dbI6|E*Xge_e5XH@)-e5eSXDzDS_1BrC3A=BW9+s$5=e9mgMh*k&hO&wT=sh%lZ zOyrWGzq(}#e0)OOOQB}5dM7P{%I$32QV~;o$47m^#)p78dBXVDp%>+&+E;C)h|k;2 zBd^}XZzPL3ygmuU0whXjps=yYPvF_Ge=-)hHa(jM3#S1063H-x^wF){=v<-L(-l3j zitniONoe~2Gd{|NUD+B=dxFKpIc)$#3SqdhNJ zLk&MESTswlW2cds2Xnb_5F$ORA*~Tl$S}pmr;>OJ$oxz66hb z;=M*|9gi^P zHxZmZ-r3PytF%X|;kw8UTLnQEHpqD!`#rC(s|akfIV# zlJ}mj2;rd%@?+}~XfY!AHHKjl3%CnK9q(@<>u~oH`R}C)3+711+#6dMU(+B8t7q`A zUVpM`DxA0@6isr+tzcU(Di5U#9u_`ZZhh&=R=Pa4%4&W$uBt9tl#ub^*ZLVsk9e{G zL{z(!U|;4hp_%!mZ!b}%;Lb)>%u>Bsowe?iyq?iB=uqmxJ}j}a>(N@NS;qNu)&cHq ztqYfxaFL#=?#VYXWviN+vjTDr7PY%!=BISl&&QugeJu3k^PgA~xL^;$``pRBd|nk@ z+m8_J-_N8s>37WuWXID zuU2@ATzp!rsU4U5j1WM*P1VpAm595qO+CQ%BKUesQ9NhT+rqV?AIFp1WQj~Q8b1|D zFm!0?r3wpqWJfD>)Q?s7n!R>8so>OY}0Cdgo`AP;I$(Rve<1vgpJiE~h@fEHM`Vyl_LF(1e9FNGi&J8Kl$QD8vZ%x%w3nY~F+t=8mBAzOEFcuecSL7~(yK8fyHj_J~iHL&=!3Obg~ zY}Zr!W#;RPZ9PM5$s;>OMHEJ77<)w{a?`$ysq~rn82ScUdvxeWRoDmRVMO(I*0I3t z^3FUo=O34sg%-Bq9gUr(-S8N zTZhr;FwIu8$5kV?PfiZMa?#ONz0oZo=t_?=u8R4QW2PEdK>o)darG_Ep{LPl@V_~nRhweTTlm zFmod)t!z%)fBjo??pLPo-|io*D~2#B4q-_1TRc}4+mY-D*n~1VPPbyYbXcG+mSzW* zP4L6?t>@=w2lZ;>y0?CUhlb$U(jtwV4*jEWRx_lFi_gOYg@F0Fo~K35YnvtSS^VVz z{#%L=%SZcTl_hF&1KWkI%z&RO6rRcuj#_Dqs`m*mXM5cuTE7!rgS-W zlor9^**B)+#EoQnrsri6U5&N{waSVCFSzK=cAurvJZ3kX{_0?EOoDSh_^os*tU13a z4kD|{j5PYb>PpF3IR7J|5v4=gWl0O7Q zA2sbxM7HtvEB#~>t)-YeVO#;-pQ^rvt69Bl3X@z@{&6U|zRo_Ph=)&mU-7%(J3zUg z4R9^Z&%FV!U5eG*=~+)#6W~hfyOxv8mFna2332_O%e6l)CR##;epH+`eBSV8h%*jM z{~8(3Qxz`^`lJszf-H9_;1(Z`8q$-hmAt8G(s^mt&CPrW}Lrz5z7*jU;r6IpQpT^vJoBd}L*~_h`Je{OU>wc)@b&-!52I z@EWH{`zqy)bBU^YQ|Wnp=Q7~2_rH?%JTa1BDN=?hwT%*-N+%S=(2QJ6bX6Nc$sHU~ zXPgS(vuNclX0ytYPu4WxI)%^R?{e|>Hy(pW4IcA7xx^E86Tb%EMx}~=Y&u`_mn*39 z=U{J-Kc^zgvH0_r<<6Nn=zD6`hT+jOGnLIN^ubkWjB+C2kB}(K3X4U(M@wH@hF`ju z6z;Cju{$|Wyw=-UJ~$ESn+}w%*^NJ6t7>OQ3kG7w6wh8N9y4o}N1V3NIFmS&*bzMB zVSDHW-eSkyS6>l|X{ACaOt*3cm40;2#I3Q?px^8N!tc%PejsgRwf{a^gJpW(;f6t} zI00Y%PK){JufqJSP)^yEkzU0ZMVQ=;3lVh2TlGTc4ot}blcwW~$lG_jAAWcbWJ4Z` zm_@;o2iYl*9flaYl}{el%uDQ@gHP^p2v&rC`|_m;Iw)T8)cqm(sax0L^j2G>jW}u# zL&LnJTj0K*(e7z$=N_ICoYFt&?#<7o!Bz9(W)71AmCd5W=x&D58N4GL4;9!|MLv%< zY|j+0j4HH9{yJ}Q``vXfNWh%P@XSe0^-%DAJ@La@g*O?#K(>cs}Rg8=LI2)1@z8H>WZf?&w@;I$1VG zcn%CNA3Wt-8bQ13p?RjyDUg`D!^I1Ys(B*53~26hsZ3Ik#^B5Psxa!!=9CIIT1}t1 zauYc6<$l!;azFX#F}kG)ox8ER7Xj|-5jB3kR;LZS7;60-KAQSA+Av|3OQ^Oiu?QHE zS4HTRUd*$+oc!-j#f%bdym@)k;~UnMR`Yh?_3mTWa{?v zzRh*yEQwKQ(}jynvi`G$rLtWE-ES}w`+d04 z#SA-(qO<2?nT%+GR!9)^g4!}nm#?@Dxl^HHa38YvA`hQ8e9&?pN*`5GeUAW z%Xx;FVmuoK3Vu}F9r|5I(6WX4Oy&rA-St)Umcjr{k7TVsxW`6hiZO6r4$+AKjh2;0gZ}z{WJ;zmc&kQmhoM$`VKK=6J2G63n%sHn$cbhpg zK?*#lEa@jRe7x7W{nrq){gwDuqkekSTaq&5vA@vbIueIW(z})S&0nW6+$%4LETr)6 zuS&)O&dIEpJLgZ455S3!r1X{9mdt}3pQ7DV+856C2bwJ#NwbsbDqJycNozYM+|J9Z z=a$iO8U+@9fA|y(1UghquR6A-PybSI6ka~*(5lWk-Mvu%o<{rAb0_Oe)v2wHy@Qp0 zr9Ojb)<)OnvthLwzkoO~g77{`NGYzp%Cuc&@TahA@QOVCf4LG<|HYNK{2x-s&@WQZC@S@_5bfbl9UvcCF=;K2xXTo`5=U3%Q^}P4N3N4ijX~oLZBk|}GpI|oneLY6ZxLYvnKr)> ztIe!1b;z8{i2EkNL>V0hgaticOXwH00p5`TcI|H&XkTVBhsUzLaS!(5Ts+(D1D_W! z8Pz+ls0+1Wo=Z-oHDONL-QmlOfwysqOu}7{XK4hS2WJnBJQT1W3;`;>szl!tc_=%qaqINQb<|nFrQ?O_}N*r&bdHu@%7v_~GRa9V3_jWGi7gp(N(#!{(M-GeJ)Gl#p|KNH+ z;0@cbC-nuR90k*Lm;`3@>v|xxmXW6=N?nmn(NYj23mZvbiyXF(c;Hh=8V6^LFKat zyya5g;$qgERlig1r~TaOM)OMF^`N##W~+~HKp29iW-D3VVjQ88&7=HM`03U6;5Ick zvb5!qLI_y}nW|572E+TUq5FzBh(C6Yg~F*sdyb&?AjmO!kuCSoHwFsuoibGJa+N|; zUg12x(Y#IeAU!~BX;tUslLNbscYli<6Als>ajZ!zS|e66B4^I71;u~q z8neI{_JZ>vbTs}k$_5%r4Zk(3xH7*S%mA89C08 zmvQ#7aCLS3lQ4FP?IqE|{7&Hv5PgC^OdHr#N!L@hO6PdiWttY82v>Ww&&xbNH>Hv8 z;qmb;HAsi&$V?;_GY{x1n>2$?+CE1x!CY12#lLK^+>xqLmMHe%JNW7)r7d{sc`Poc z(N0Ip2#GJJotY&|!Q_X;7e|!uDBBl_UXUezZ+jwii>Gq(g!q7ZU;l?6j0|IY5WV2W zhB@sSZ)Sq?99s2 zr1HZkYI2pN%VPJX%VjoaATx_@X9Nfk;oqQ^3AmHwF{d&BxW54ai7${$KpDhRNAEm; z2~WeeSURY3I{5Y_B)>$y)#2^fB-&+|adkf9dHFcOPqqo(B17wg?CaLu09vdP3r*U> z&cot?jbX7-qJD#w?r7X!WQ^3NajUAWPEKx(m!jG@hFCxfAV0Klt)hX34)jh_)UMW< z(;G*gj^cT<)c{FU9;biO=P}8JN^!@`70s4cA_6>~$cMvB94g}%o>nFcUwl{4<{r*# zzNYs7040f56}w@&p_{{{p!BK)Y97!~_2f~8Sc0gu`R2Dd#0daZ-U&feu>4elgf*NS zKyRuT>~gK@NaAqzZ^Uaj?JUI#tc1_2+BPIYnJN#(X5FvLjEZeqXO0mqY77nMeZ|b;Ab9sT`+SVX5f&FWTJ+H9 zFQ#pm&j`jmKE-<>LsN4>4Hz34U{8vteB)X%EEF~51~x!>vl}NmxUv8dKk3|H@ze>L zUzjEG2E14N&qdaovI5gz3%qlSJLO$O~}R^&sx= z_}`#Np_-ABuN4tF6=fHS6Mw|$vDZIkq5;6w6jju?Yb)v^1-!;5Z=NPhG5M`r?t~A4l3(k{oqvvHE5pMZw52Gd*%@M?Ynfe6TQlH zWMr3(uP&QILD8NB6ymD-mdUq7M?|)FNMmn^#y><>g#W5uz)Dx^d)rf_xCP-~bz%$j3^OTy zfoyN1iG75v5BJJMWxoG-Hr1s5;FKb2-TWT5LXypy1K`raym+7xy?PtG&3I1=$KExM z8bzU-#Zsp`u3^M!|9?;_YPS&K5`9bo(-~dq#4`P$qT?elPfUlaQB+#_eX4in54AAx z#kwnm@J4R%&wjd@K#V$p;m424k0qmRx;1wYHH4b7J48nzv`^Aud)S}35B?kbUuM3) zo$@GF>esZzI~2uwHG0*40Q5T*)or_XS1Jr@-EZ0-7-+M6PsK^J~v%T{gPOv zUCMr?my>6V=4xm0h_P{ntre<>yLR3f6g-Q+JxZ| z??XS=3R&2EEqG3s4P3wYe%l0I{9qALS05{tM>uS2IQQogyx2Ln^WM^L2+bza^?`2` zd;gBk$xn1|I~IebgXtC;lp=okBQ5%Q?io053Lkug_lebvyVW3WL(Ymy^t-M00F7K0 zH*9-us3O0Fdvwxj1bWjgTcZM5mo|R1b$$S0)HFAmn}a`WR%59MG;ZTRJDA*JIwVtD zdL}gOBJ_P(_@n!t)#{xqwQ)QopOywu{PbwVLejLo>KOr?mzScM#`s%aul{?dJ5Z|l z@aC~%cdS{%h?l;u!s4yTFIr2zg7?O{3f~ld`!412g=6sB2)A7sE3Qtd@O=u0Kp~m^-rV`>-39hM3b)7e0 zR|l25PnDb0kIn`qyVv2m<%dT)oV05C(Mx~siEf=x5m9vaA?(A%@=h^3U8 z8nF#RdX7l12-S~)%?P6K$C^Zc>ER&rHr5{&HoAAx-TDb;hL48ci_OD_H{Nl2>opnZ zaw^dOv;^vW&a0)VxsM=JpDG zMgM&%d`#X)Aj7lQ-zlU*T)GJXmUTbx>jo^LJ`-#{D7SY$b0wF26kd{q5cx4Jn4)LA_m1 zc418tb8_3$@mTNMll9QF8%AdJ>s^vbsL+OIw1b?XavP(O1GF$UdB*3?aUPr~S*An% zR@KqoTbuVAgc>YvlPKB2y@ra8+GrbNydozCrynHh3-N*q6*nQdm~!lvp>w7VbP*AI zL8D+(CqldeR$OW35L<39rkpi&O;$x7Z!*8lQ)HFGZf3wX+6&W10oeFbrLDULlZ>O$ zZWBB{fXD@j)0}b(;@TuFs;g zlLw~Ni91^?6*iE`OV!AR$qI`*r{**e(h4+Q2kb1GL=W_&zTco!k>4Os5!#~bPPzHP zr@k9M_QY)5D$qNWY4=&zk@S9tsa6NuzlK$iRkJ6o;Rl5z0?i1nOapHL7kXiG?)9vC00~*cn!M{(dVKt z{|tNJW4{CW-*8P1M%lNsZ$}G)zWUQO=3)EoOA~*ZeesX8kfZI@#9TdnUE2XDuCKU5 z*^KT2XYF4%zbZhi#?MFU#?RW4pbPI?#$CPpSz$MPuWqt+{$S;oryHKz6&rkxd_J0_ zSop!)dD=hIi9vXQ=(K|PUy%htzCm3GcDSR8)1p%>je{bBpRPBJXqFA=O+(PuhXT9{VKiQ8)5V$$`9)LI0@X7D{|JVVFRu4t5?2!=z9yooLZ| zIWX;#AhX*}Ltfs#5SvwXQ^oaLzJwd{#W&Zq&VP5F6q_q|s3dqbv!JIe`Yc{<#Lqjrj4iIq|B z0`tlNRAR(2Ag$xcfg$i5vOg;Iy7PhA5=kwrgqf>ys~^h4Fenn6$^eC^%n{HbB4fWI>YvZ|T*n z=+$^D6A0bTVpO#zP)GU0xftU^dV?Bi{j+HKkOsNN)T;|6A&aL?K38;FgXQhIbzZFU zKeZQ*tNY3ud}{Hy5_~HsE(E1MGugTWP8et}xAf7opKg3Q@2Fp-)^nVI0UT@YYGbtz zsL1{PGcMe5k-7K*y!j;YrdgC1F%EYPZJaQuUFG!QHdd+jDMnokFGM>V@8$VIl_|Pyt9jy_!1`1ESDeXh#p zk$H%RZ2pQ#;gj{fPTPTc)OG`^Y?vx7mv?4oX(8R#^V}4{y;w?R%0?6pP>8BXvJMbX%2b%fO>lc)!y+1IST2*>k%ncrxW z_|+AtprzuSphM~SU}FifetwKBuJO)uXq6CmdX(_9rTj+IF*qm4^QW@BS)82c$Mp63 zNm9~pP#T~h89!2CNk!f>nY|gmhls5pOJ1hD1a`*|<@?H1ZmeF)Lo2hVLK#aP>w9!D zlcule_EjSKAg%q(YqS&i2mX@pOe(4ux#98DrrsX4X^m!BQ-|=HGo6 z=Sw0MKNowQElNX58xNi3ah>FXeO^2_;$s(_x1RAsNJKIDH;AzgATLh$<;^3Xy7oGH z`EgjKrFHhCoD&3H2zev-vAz54)++^+T&(H^lgY#is4wLZx4QAVxx!qc_O)`#Le<+k zar{Joi|Tc}Z0h1T00K73!i{COLuAuMH3CiPK+Daq!|bpcj`SNe)8x-wJta?4Pf z^JD{-+(zJtNyK7$NYf=K8Gy6#j|-WrO*p4SyTm!5w|$gVs_?2d4mut9N_{SMM^zXm zi-!bLjPZL$_|{KDD`_^RRlWmb+3!nEKQj}#nHMvQ<6Vs;!bv9l7vbk!8#rqBwu{l^ zBQ#f3%}%VJclyt+qBl8OQz28lA(73MB$oM+VVQb}@u(NnjI321yfEQf`|XBi6||d` z;dW{4Svxmj=37hr2%Ii#s!QKx8(BY)H)F{gKM&p4Au>ITu<^ehdA?^L}Z%M!r*Nmr;0WQQ!dO z6f$ofdTD+O0tjZTdaqrzLyEB|-2TwTD<%4-t3CRBI;)mvR3?SvSH-k#%i|Grpz%+Z z>@TB68}h2J&9-ibnKEH+wVxymMZdKw?n6i?B15% z=i|63)}&CiNbwko87D{Pb-uBC+|OfIMNfBwelS)V!tB3OGfZ*H(x^38A(hIQl39z% zQ^Fs^RNhG#)?OlE^Pa0R4=JmlHfIFOlbxyw%pW@MbaWXpi<)|azP?dc?3D45LncbecbcPU|mUS<@T!d?@(W&G}&kE_<~)q*754I|fIn}z9_ z?o}3ix2*Y|I2u2_*}{WS;6)`|TA4b3$J~4lamu%QAv_|n330w+v0&qcy#5s>aim}$ zb9slaDSvvY^7Whd5Mz?#WTFM$y@}`b==OVTo&BAsi^iWBx2sMtd*^#DUpqe-w6~W) z$E2DS=If1A)Zno-IOEDz@e$J{Sv&nAV^j39g_(EkbYBb656sOuU;m!8OmxTn0!3^8 z27v`Zw=k}v51!c%e{5{x38RkS)hO)pL?coy`GF-&p`K*p?xbvci8e;|nJ6)vLQVF5 z2)DEIacDl;UmV3)ZW3l`O#Ng%3Y40GYq&fC6Ci+zCsnV}CTIf?09^O`4SFhf?Y!H! zkGU;k)ZXg;-gM?bGeIJP%Cy@}qiWCm4NWi=E z_zj}AV{xG{AF_pYP=`0rqJfUoVjAUHi<=#z`9stUg>Y6uKhLWz#{vq%#sKVY_^HJP zs_rcgS#P*Mb__)sUv1Q2kID;AuQvIDY9I%R$Vrsv>6!|n?Dq{#M_&gr zVQ~}F``m$*a}Cg+MHk&vPOS-LMrNyhf1SI{j+K@k<6z?DG8@u<2ppESsv#v~LydOc8#u znL~;v#?@vRStoy!=@pJYUP#7%DxR+jjEg)jRy$$xMfB3gtyNVsB)$j@rzB;(*vl_4 zxbdQ-jyOcn-d@g`&Y@PAS4v;)t=r&N6tY{}@M8whIhFTC>_^iCjYVMZ-a_S2Zc#rq zT=ISFJJgP?vH5~TX6Y?8#@}nWo2<3yk+k@My3EA>WsKR0rx8-Q`ZG<9kCZsK+K!=X znm6QG_m|y9+ErptSy`1kyByjYUC9B-c$72hT!s~ApJHEV4=pv$-F&3eZid!1xG|ZQ zd|35)bWha9I>DcU3&N^tdx>kc-ybSLLk%1&%gZ8GQ_U5Cltn$IOKTuutmGWa`~*EjsxZuwfEsrtA{$8ZWEI~(`5$~|l${9RR!i#6_we=@>7Bjm$ZE8U#Jz?fQE4(fTk%l6^Sl7*NO{~(Zx z^N4;|jCKZq32sP}sJ}tBG|}In;ow2cH*QJ=^}}zFz2Rk;FtvboEohLt67wW5+xM62 zEp^8eT@p|3#AY!ZbDh2ZKvc{0dk|qUlB$Q=vQDlOoxfp_R^tJQKhlyqFYAy~-weo) z9;~SS2^#`nuD=n@f8L|gX(~azwx>z!Pl!bW)c6J0$3#n)UjkCOif2R#EA8o&>}Us^ zKq5rT$np{Ly6SVxCZ+Cufv&W5nYs2}KjG3mAnj)%Mf+MUT#Y~{*t882Z|UbOND?$f z^jm~p@k%W@Fh}otQJH)<8&xlm66d=SOuktCDC41$kHHBnpLxpL7FV?SKHnD_#qb3w z)4FG*0C~~`F?x^`)kGif`NnNa`_PSzuA0%AGdggtj($(Eok`<@{||QOgK%vcn5IcV zXsnh#=6W>Z3hqAMhHYwUGY?p6mEWBos6P2hRF^XI(!)*4?D-;~!pc4odHw5&NKs_( zl-W1>Q+`+NO4~v_JATfI<}W4ayN9)yAdMd7a4Zq^Jmi5e&HM4$@xhtV3BDkv!??Hb z^6*$y<>}l^cYKY~Uv4^f@X`2Ezm1kB$1KD^YQ)Y#wK&Y3urM__ zQ8JEHR;yicdxe5&S5ZoN_rlw_ST?Tadn^3F_kEy9So$1|1 zkR20M>H2(^f|v=)c?4Hyf2;$bV?{mt->U-<^~Rc$1uYa=5@ZhbF5_}Ya}eXuU|Zi! znYcI4ex!i7oM0z*HohxJx9ldB!N1#oRO<%&6G; z4h+lcj8v@Rr_4Z}cR}W!n`v)?iC8*LBIZyORHmBsJdqaZN zF>PFDnQ=nHuVqQ7f3}zjwNgVMRE=yvdPJN3spp*Mi`9ys+3p&?(?*Q_4XVkw8vms- z`W^i9_r{A_2kZmD7FGrhS93xAy@Ux67-m9&wt<<1Vai&0Gu$U(F_TsMlS|C@!`Qu3 z>XKX0D=fU#eu^lpZv}E%DQo}5tK-oNWIlKIUa&280Z!on1H3V{{Ws_k45?%%pm6SF z@Hz(VLlHzw@Qb9hAn;LvweCqSRjNLdb;d(M zQ8vosmG0qRtd$FzTyRHOTZPv%8?5((>n0b$Z=aIB;Imq1S(YCfU~;A{wG1^=?{`G} z2F*!9pX{S_GSOL(@Lm#>Bt0+K*Q!Zk_ezYimF2J-1I_4XIvV29mCV8%3nWJ zF;qC#?-Grzx|qnsm83zbG8e}6(b4CM_0vuU^G6iI@1nAh_lurEvNo#b-% zgVvAhPec-gZg)s<{q;(`eGE5pS41hU0JUZtNiLibczV?gt*I~JyZajy9%zoJqwTeY zY3plUSn*dn#^ue$b?rRE0E11r1@q6m_x|@FbQ4Lk3bnO9T5bfZSt^LI-z`{a#4e zMJg;hYI{3X`*amNWV@!v4|jf!Qajf=l&5veBt_Z2BL{h8RgXj;TTkSg<;WdFwc8ju21ksQFR$ltYoBz)+@bqo29feEYjcOz{YnYry7m zkUY{L$GOnCuxuq^Z6wuW-$yXl@Ql{k&*Z{xN}p#FO#sf}YkZ#|Q{y!ncdf|zT(tyM5&1A!;{HF^kM z%Gn6$_T#!Lf37Qb>4;D0f;lZ}UHNXtE%VE2={LlpBDUP-68eY(6Jk-$zuIM_uCxec z*#)zl)+J#oLFDOzL3nm8N(Gzx zi$L&mpU63c$CK(c-AR*88MjF`3tD*iPbHQu++Pl@qF(@h?7VV^!`v;^@RLp_>09m0 zm&Y?qA!m%o&0`P-y*`G8%5Z_q_|Haq)pUO+#+iSIagZ1mcjBsMoWh$1D3T}BB+#Ef z;Do`cN+t?YKepAGFh3iU`i@k!Av|&gMVjs+We3!YwSlaI(A`#>4Cnc{bH0Cc#8FK& z(BRUj*XLbWMRP0D+l0OQKLH)Fw~)x&P7tk@)(-!_-r zf5^zbesCPd$f&`nKiDE}y9;N-boxoA1*-I1v;DTyZ%zW}EisLuKLX@GH#Okoy6%uf zAV2x&7j95<)a<$X(iS8uRHnD1+&0}FAd~%GbP7t9@kWSGGI_SCi$TNyyv+>AsO4;- zog&e(v^tBg5*}3EX&QE@VEWEepX%}X;SC|Cb~i10`U?*&fb_molDCQ1WC{GKpXB^;4LtVi6rCpk6{uEs(#>O~PXnC~xo39*F*yM?Jq$2WtcrzWq+;q(h!b#e2bP$$U6dD3{C5t3g32XyF%75=s zsU#kD-7r@zr1MqwUT-4FQte)!v#zhR7Sj$O+{uwK(u759{2P1mpO_3V7p-j;4OrR1 zk+bO4R<-fPpUM6vwp{pEn~4_P(373My?Gnn#M^#lGJVm^26|6ryX~USiydTQGJtQv zoRZzYct{N(0gKP0v%0i*u4JBTSxmmV4};5$fFVv*!p+oS)*IkZ{IwPT!`S)H`RyNn z(+88{@ZC*t4rN*SP-Tz29O}pLSK!*4Qs)yi1(tdJW2bn7HG||}Uet7DPyEFNn}=uW zH0MUWcfBqz_KbD~ZP*mz>%Ais%+#iZ#e>!!Nfj~QI_bb4s4ZBS?L3mVb^F3fxLd)Y zVHO2!(x+_3BcBinV5hF>R__H*2EN~&R!k@t1fTKjMqFx-` z>?KQ%QBb6#NSj(1mi(j7ukV!V7_|&Omoa+|?t+eMHRDvnNB9y)?zI=G^Y2${f^Yg5 zodCX`qJs1FQrhfC-N5I(OHG9t&AZdVPU0n7YV{#);dd)XWLBOd^XhApM#V0ybS@vg z<6?f%69g~4Up&C&k82`lY_eVOutI+^^c9cL%3SKd@m zi;?Nps%z1G3EOL-xtg9l(NY{-6WaM}F0F8dia-)zqIvUA?D@~%c+7YJv5MqJ&;v>$ z+zH3nTy@9ky|SQ}P6_%32OWF!`r-HS`)uF#RQ-4rrpG*cgWj9(%>x#(Aq^x@GBkoz zeJfj5hcZ}LK~(tSo^|l!YHO4j51eyINEohuK#fRvVz0rd`;jA3Y*+1io2h5f4b^!d zDm32{{0}S7f4jB(`h?NeKZK3`g&h1t%FaBzZVT>-DWxOFnVUNaB_NrR&gDVIkcQ6LHREl3LHg2gzvzs)#Dz~Jbgc_9Og6NU^h zHbR}&v)xeP-A@DQH}3an>YbnyzD0k<6@UH~eitpeQF7w^!$gw#IDC0?EwIlRaqr=B z$rjcYKxYi|)==F5`W=i8>I+qepCjeVDV`Yo`6dYC2JUKT0YvTKc^2-Ve(rFbboSbD zjjy48WkXH!4d?=As0{ISic4`XhmJ;-)Y}bBn3>SKJyD%IAB9VBRmRx7&-2UpUaV>B%Fn>&*m+2_mh z5!@*Qw%7rktR%Miwfw{hS$DlmKBGS!Ogm_>CN##DI0K!Tbo>-l~#yP`B z(6e4ok+$r5G}uw#Yn=t{T~p{-6aYW5{frkOQb-zP4Vdl_jl-Ao9)X#=Er;)Csme04 zu$g1xTGHWVw;UMGIQq(J38Sp3f`;@vKAne2$(gaP;qFQL*5ueolILO-?g$@F^dlwZ z17SL&p$l9GF>*?NjsZ$mX77+adZJKio4<}GS`^j$AH=$LS0G5Dn5~22jY@ybJV4tg z(vYekB<^fq1uJIvj9TzW?4vt2G7Q;MY%-9!>Xff1+mh!b6O?KMk){nH&}wht4ghPU z1+YfXo=^1lQ?9DLAd3(K>m;kTR1H6`1ZMi)MyPn>re_rv)c_Xjp9)9gu=$%^73{jN zg@<J;f{T6++_b zQTB^Dhw~QkR$-(f!fd-7xa><6P6%(_Y`DOJQ@B}ThpsbsisZpwVDyE4PBn|wdbk#= zFy#|dNAbba6Hky1$c8Z8N)mzWq37w8sWs!+@b+ad`*gFAqc|nBeGE$1w{+wftE{ZljJ=-qFoIj4 zxiG(x3G_$c(+dA)2mu2KxbKAg#Z4p`u+b>AO3IkRWP{R@9L+%Fw>GkDbaRsA>$L7yO3DaE7-;^lmp+>!O;2h%2&XW-{F%N|3y6>! z)#ej1j)-HSAZCEb3@`vpqU^II&Uv-weqhA~+D#}R{@`hbOaz$&0A`uYrYlQ@pHdx0 zev6PxO`CPw!cC2}>H9MP@97X;*1y#b_mVw0Yz8w|Oa$u(XIOeI_WXzfReL-WMl*+iAa*1L^4Ruaa0rp5tJ-ZKtu#YB?%%!MzVlp z2?HV-W>AI!hM8aE8@}(n-#P1^d)NKvu5}Bl`{~`gcGa%9cU1#nim(XKU)9#r28f6N z;0gEx5RgESR+xt?02mkmq5uGp15hFkfCNM!3!p~C`8U=k5(Xf@>BImK?*TyncP}I;ffSyVO!*49` z8`6J0n>3vQyoavep~&9CYQzwh=*jgoH64sh47K&HY5i7&4(sUWcbk|V0K9#I0!(x? zcrC50dC67)QcxRufD4dybPDuSH!`|*D)X=VZ~1>;FcZJa4va~h%KDr9zXOcUE`d&< z^0pxRE$2WdZxFWv05s3ZFCYj2NKff}p+SD9_zsAf1Hb};IQtZN2hke`^#DNqxv zSAZ477n~ih=!2LG#M(Z$uV4F-*b1-`%^xIHp2ae1xTj^00@7GhxzH~parpH zpqJ@sdBCzkR3qFqPwBsT0)kCW%g+zeO2z}ZU4Bp($QKtM^*`$b65=Ni3X9)we88(I5CLohJ|Iuv-{~}dzjXmZK>Eqw)IZWn0It8^hWt+91^fVW@CNjO z+u(CBh+V)mf8}NldU-_aw84Mn$V>NE zdQvUYE2O%l+N2^-4yYPb1$qshEMh%e zyH0;=0Q{>p{^X;$KtW7#hT<}X@IO=1iqgvcrSI>${`Z>vqpY|7uJ6A}@Spkr>x~EC z43<;>@7DOU1dxZ28b}YM4blr~hBN@Y5Cr4{WB~H%6#tpO_V3np`#U$2-{tWF+r{H= zp1<@xZSm7QPy0vUg^CO77jS<_333Sw0{b(d>FXCB;Nj*T#H$Xj4lcZUK2DP2yfPPL zqygY`tvTfZfS;_t*AODszJJkT!7+Zxhd>}S|BI%71OTt065th31-u6ufM%c#=mz?MFTgl33-+EBU=u(A7~q(Qh=`1chKPxX zors4>fJlr;hDeb}jYx~=8j%T+712#1HzFURV4^6ZSfWQnsYE$MMMUL9?}!?S+K76I zMu?_~kVNZ5dqhXX5MpX#X5zENg2Yn9ip1)~`ouSg?TFoo{fQ%p?-M5zXA>6_R}$9~ zw-NUfj}tEvZxZiA00<(D0a1i#LJT3+5LZY5BnFZQ$%GU`-awkbb{>N)Lbf1> zP*Nxplou)i_F!G88Po~t4~>B)L35z5p!LvB=ol0UMM3c-G$dRkVkF8WdL))4ZX}^3 zaU_`}FG=c1y1=otLV_VBC1oWQB2@s#q7|tpX(VY9X+CK+X)Ean=`!g)85tQnnHbq6 zGD9*)vLLcJvS(zKWC*fRvK2BcITbk%xeU2Bxiz^D`91PX^4H`o6D26FkD2^!^D1|65QQn~Rq`XU+P5Fkhi}D-gPbw;^^HfSy##A0u zcc`+d-co(0LQ)-4Gg6CEYf{@%2U90gmr=J;PgDP-p`j6^xlChC6GW3tQ%=)CGf#u1 zWuld&)uVN$h12HJ*3*vC?$A-t3DBw2+0#YPWzyBq4byGWQ_u_2YtlQ?$I$1}H_}hi z?=vtlNHZ8Rcrzq1yk_WSSYsq(6lBz5bY{HISi;!BxXc7$;$zZea$<^QdcoAiw8Biv zEX1tG?8*Fyxr%v+8O6fLa*@THC5$DPp zaOTaK(K82ZoNSlbT-XxXs@cZbj@WtGwb;GbpRzZw&v8IGL^+H(LOBXJx;VBtnK@NB zoj4OY-*e7z5p#)hnQ%pNm2eGk?Vsg7t8>=x?6b2SXScXnxi52ja;I~*aIf<)@u=~5 z@TBt~cs6)hc-47t^JepQ@}kahozp!Re6Hx+(7EIDLg&rS-#cG@evXflPm#}+FP*QQ z55>>JZ@?eLU%@{kKp~(g;4Y9Q&?9glC?seh7%$i$xF*CVq$d<1R3Y?Dm_`^T>@Qp_ z{8fZhL{Y>`Bwyr@zs`NeSCh0vHQ5h$hT$!)3G_u;V zaM=&C=!;?(T`m?}oRVXbGmwjyYnMBbSC9{oe_$oUvj)ubZJqISM8?SbG1d-IhZ4? z7`Ako@3QmdmzP)7MbtgjE7f;2E@%X3)M*@ODr-h-wqAi;(Yca%Wk`!r%SdO&RfVh3S3C8o^-cA2^p~!QUJJO^WI$|i z)gaB_+jW8KKGz!z0Yg2*G{ZR~AtQgIkH)0NM#j0uKTM=eqD*>BnM@r_Uz=iXXxvD- zF>5Al7HrmTPG@dwUT%J1am6ClV$o8{GTO4=io?po>Vq|zwYl|6>wOz7n+%&3TRGb} z+X*`%yD+<6dk*{C_MaT+92^~LZj#)zxcT}f&e71Z*m3`s-mSb_C?`#)ET>K9%g*V} z>n@jEQe9SEFS(|=uDM-uOLJRyhq-6EZ+U2X&(eQ-u)d=l~(nx5eQ)GM8xv0BQ$Y}NGq8Or>TQTi$KKOn3${n3M zukKRb^}0KF@4~&*d%t2WVw>)ty?^Ka@&la*6>+q20dbS@D)EI6p$|PC4kgGYJWIeQ zIw$r$y6`CT5iZFosqeA$RPppzHxh4(s#&Way#?L|zukCu^WDUI{rBxPiZyR)1#0u^nCcSi ziR#1aQ4Q`5iyy2$j5JwE%#dphzP`ft6%F@n_Js* zyF>eIhh@jtPUFs@F8!|0-P+w9J?cG(&zC+o_bT@`_9^x?^vn0x56BJF4ayDH4apDH z4=W6R_@eZsX+(9TWfV5rHg;vK`>Wp9{_*SMBNJv5Qqo2#1=bA5G5Lu{RlwbUWyn-B9GF_Toc3R&29{3%%a{mX-kIYq`)$%o&wdQs0 z^)DMX8!MZBo5x#m+YH-zJHk7)yO(zdP?o3_bO4&Lm-LhUXW6fdzuNbW_UAF)nB#*) zEIanqq2giBk;T#KaVU-qmwh5~(uCK?eCQr!PPo0(b-7PR)4DRseVlM!`FvY%rdh@HZ|4 z#Imw~N04$F0$pWgegh)v0MInS5(q!f0svVP0GtF62*>#Z!pRHJpO^-KR{uYI&{J+w zdH{H?c8bQ?EQ?O>{~rGEPeICGI{xzxYA!2#@ifVQxf5Ce266yL6a*nU4-hjDK^TY# zod6H0GYM$WfZ?x9h=?Ij5>hg93Q8)Fp^hFPCW1hSp%4<1(+vlrXz)D%WguZZC#_D( zWb8FhEJpNs}_F8YOU|tEwCG(M1)r=5 zise?o2qi23MWg`wRJGrl{qGdJ_y0(0O25D0{XlmrZ<oRKTkC)!rFQAW?;Yl%UVb&d1SpCI^coEM~i$}g{g{1F%U)IvI%?-AtUP-?| z{W|wD*^i;HN_>ssVb;9E`z)t*4zaymJSQCUjsTS0$Rz+&oILm|CI3m;yhs3^MI*OL zr%tXEH}_x1CSVZp6D`8cwGFkCKSoFQ4A>T~46`g>=J%3SY)^e3vpX<*FXpyWB9`y} zLKs4l03aJcoq3Mr^bn|bU`f7TJu1XRW4D*96P4Af;ONj+3e=mO?x4h(nP}Gq69xbG zJ|2)RE_LFbyzQWodds^eLvCpk5X-I46 z5~{>a7JC_+?_5Ud_OCRG^mA>cBKTU;11(|7$kmynOV|>%0ryTbQm$ zc+c)*yfoGk=|$s)l)A@m zLvN(Y^;2_W+WolsIqKBUQ+Q@A{$om9#YvHxXp|~ip-Y0%4r15v$?%NFleu+48N*p# z-RT)!>~|9A&qQp2T8|;!T}RS;WM zlq_8w>&B99QLT-%-wsW`XwS;w19a>;uFRhKRhFMAI2qq~0pq55r8C%@!+2e z+brl5JFE!ulT>Gp`BT;klDXAsx1Y|=dk1{V%kHp9uYd@X&uGPU3k;i*wP}nsBHO9_ zi_vOjZNk7m3KmJ$VHhV5qptM5eLMJ*g^kTLtHx1P-?t_fws!i2I}uNZ<$fS|VD2PX zy~RPQ&Ze_@)|E%NitjqI=?;8-KRFDu_?LyH@#BU|yD4w6@I{uCZW}Sh)yA(kYo)DK zo)o(0-|J=flerQW^KE)SkFKW%d4hYlF6dG(T{n!@WV~8 z6jg)ZrAP!THbp;kO`E2rFy@JkQ086IPHAcOZbQ{ng8&$+w}poQ6diQ)U@sctrR{f- z4lHUViv~Ph^CI-_qn@mOcGpvQEsH#!sEzI>KkK=)8ESwgjjfzfD|+E+J~{DjHR|oP z)b0=LN-q4gN}Kjkfz93bqtjit&&nHkyu>JkiK{NiCRdwwHgg`UxX)9f<+2jCqa2p( zuG%oJc<RjO!M}Gv?>4 zLTZ?!st5C&Ua<%zszyFvpXWk+8m)W8E;aaIZZQ2;*tdZR^hXzZd0R1&p-VL3@mt$y z8#DAmdl0(1lWNmllH@z;VvkF+jnSJg0UE1$k>f*_NAK`o((7W4e@5G%IFEmm zc@Sf5*#XW01}Fk>Lf3ME0JIgu(TOl*S`>@`XznJ#Q0|4eTjlss7}q`lAVNIA$)YxP zS~u{5c|{-cGj+F@PNp7byLl#1KaanuJ4(L4#MgpFx6xu*CnocFBK`uSah(%Yo!aJV*>@&` zby6Xd0NmX)?=UWRk&EP!BLM$(O#Uyr{@gdu8MD&w9Kqw?FU-xk^*#7*mU=CC>ct9z zNC#7&aZ+;f6!SS|p#eUP>-t#_k8}-nT3`ul-sH3|tf`P++Mk9$DZlw@EbAi6vn6|_ zlfoc4cUaD5=Ru4FnmbG4OA=?a+6iyHp#(9_!&|pxlM2H5bH2m>(Pk|b#jjHsojlr1 zALt7dkHDXlWf`Rf_>UJ}!YW`wMw(fcyrh_Bu!zp4^m>85*#%Dux!T;3G)CS8JxGWS z-xHb$vd+SIc?ZT{kG#2$Ex-P{QJ;5RVDt@1ZKcNhZCBw#hg`)TBe4hjY~-tv7b|RD4f81rba90=s&uHxHf7(A!nh;rwHB^X)!p|q{Ha&p zMHjmjTvHv`3Dx6M5ckicy&%?9eIK^?tf{`@O-!c)&Ax%pp0@@DzMUcynZRM?;8Pt| z(fBP>(Fo{W1jKfZ@P#mGO5x%=|5Jn!M*guB!n}A6tZXng`SU~ymrggW zg6{;MM=8XPh)ycu-zQ>Z;8V|*cuo`y9ft5#OVw2C+{!0el5*a+hWaF=+92D$mpc8c z5QUDsj|RnC1#yo*zWd0>UY)s2e$`UsRmDd`Q24;UOt^rR!aVHghGElMr39pln;mUe znKgfAd~5%adtrb0?H#@kT7E<`*77$wp@x%a z-S>5*MIcHE&055r1}<(a9@hE4+7fMM7Su0RGP8QWO14}izD@?X))v@3Obp0S;%|N? z#$fPKM5=k)s}1HcObKsyKxuYjj9(r;u*?93kKgf+| z=rCNZQ#BA)=o=eROX_D{?_(@gk}yjdOrh0jxX;?ikY zo!f!CwwVu>s1Q6@GEkqb<8v;JDV;Sh&;)?l{QJuZn5HUN=^2>eb}5RO0XV7L>@Gdl zX^G{*_}nkjsoX*Pwod;u7Uxty#iv%J>aE*V$0OVfAH>jM@A{+bhmAfkW;T5#%U6Zn zyl-3ajNgRMgu!g8T=Fs(hUMs`P$H_Z7Pu2u8&Wt|b%P{=-9B%Qjyf8|yOXN?1BYf(f|poXtv-QbHb zAncCUkwAxS*QEgqBRk!cfIG^&cSA@i)qywK>PpoD6Dqec&cqP{$S@g_?{erp~5QAI!5bu7I*0TZ8IWu~Ss%N$^>E*P=T&&OI}??Y*RBRwiG%Co`~OWGKX$mkBfR8pCOp#td@{XSGf$_jJzc zm7%EktUd=nB0eCMPq^F2f3`E*LLek9?c4X;*m8thHCvPh>Y%eqe~C4hyA=6^q}ktd zLW8V>mHXzcfpDjh2jSUK^OYV(rN#!YG+sD-lvo)k!Cq|6j5oN(Wm~4WSJ5XG{JxF= zFs-J<&6A;qdxn!2m?dMx#BPpotsfZ-WaYSZ%!SPSXt$fK@9^ixjHdG)AD?@&e7E7!VDvT`z9`hjxxj>Y+^oO! zOqJ%!c=SENpT9Qgo1*U*w)agvwu5h{E5Y{1ChDRTP{$Zc)bO1s9n{|J63yem zl!O_0p-_V+6T^JbFXrWM^Uk>hVXf9TM9XoJWfJCs3%DZL)ax2kY>wu!0J zN4Xz)CjRur+9JqdYvErIC^s(x&?8I$9`vJP@$vao1b{RIfeS!L;s-hjz;Jo=z|kKr zenFa+{|ry3VLV=_1wVjVNT~f}{nG<3IEtZInfYYn?9&5L%`@{hTuSM<#EAD6vqFbF zb&T{0=_lzrX*TJe8e%`&0g3%)4mYkI?ESnuGqj9B)g5LeZ?r^xL3}^@Wq8ae1^eYP z9_oN2r`B?;C9GWBm$rb{OkM)t-RmaBeaK;sEKC(iLZcDFLTdz*#rtykL;h>w$ z19H{2r;~}tv{W7ACCMfgq-o~UJ2QK#Nk3k=slTelyPt>=fP1#^-oz+b0`OrHUm)eR zdvxf85yd)rq1-T@sI2y5L&kNyBnOozaU-W&bMm33pEXjqAM(LN$d1O7}ydCHp&ZH@9&J&!d)V38~ z-R&^LhmoADY!SdlEFgWiz9SfLa>z>@lh^AIO=zu*JB_+XY4RQK9d_p@?b3Baq~Sjh zeid!%~Ky6*?o zPKyiE@fep;a5oI_vM6M2Vd_8y{GeLrXEl!)IP+Hj_7iP`uW6SD5k4Kb@%t!z&7{86ayW#=qdY@u+5~{uUFK}mUmfbqi7YE6Ig}Q&kMIg?vO+kxS5Lg zZz#la6=Y)sNp!!yI`~o<7*%8;8#|)>ybH~ zS_Nx{UKBWfa6>hi%9BhaoNi4M8Q+%OB4tNat~TQJ%Ikv1rW^N=fO(@cwI%L_)cKZk zpe<~HZ3Fl6)JxzE9pc0F+o-v|1N=L#oYhr+Wc6OpKjIMa;bPgp5D1rrKCGbE2qSpH;a{P8B$n+$4J<07%XxO^Xd$0lj2 zr#RjMn6VmL7~?E5rDK;42dn)mS!Y?xsk^N;zEySc=WVVU*-T!yo#!gXeu>1dARnt+ z1EuKEqdo`qV_A5%x6nfXrm2up+f1eF$`*+en^){O zJOmRB*lcXB{xGBL+ZJeUv?pqXd(%Rp6FTbz4q^cdw1}rtkZncQ#6nL%J+T*ncON*>q{3k10cU zGGoPSbIPRu!nW`SF+ODVJs8Sx_#TW{)kifH`ElFHvYB;s=8&VdK8tnMbvASQY^9Fh zY}?#sSKtCOY%v!`6eET`u*5#7+{nQ^i>%U3?e{G+bfKxd{j?JPQKu#Zc+lI=ttWRx zqouJF74ZfwTC?goxe0dSrg)*6@8lU)j#~Jo~|*@rYTBk96Rv;vNKfmP+>T6G?c}{s%doWeG#kO9 zi{icfne(?wo0r{%1{>0!(>&c{O4}2z{(_b5q_(qlQI(YU=(ZMeiTw0t{FD6VPkUD1 zrzVd@ZWPx!WSJh36}JSRq{YbUA773c!joewBAoCe<69aI4ey%&>MJxBNGfA*Hg{%v)A#-G4=@%`H?m-VF;0ooU7GYAEkyWy;h_sb@fQ0udj z=1(RgbDL#!-nZ4;^v!rHN8w}Fop{=LRI)5hn*!6WiiKe85i1|p93OVJ=QO7jx4>e) z+EZeAd{0sn<#t(ZB{!o*DTv?4#lgCyqNn!P4_&kM- zKiQrXK>KE_$^n*!NBoi@k9#T@QuTf`nk8F&D8L^r+01U%%h=j}Q_;UT^er>=mLC66 zPEHGt%F?_u3d`9or6W}*h1XWIX{jzg{H84!?C`1KW^tkzGKn~6?M*2cDd?2`I-jp~ zaVGR~q);1dhODO$*u_=x*4H8R3ZsXfiY+^;vPmSUMrVv^ioTwhQsa`u0MM_3yXz9eSls1oEGS9(Tda4h!F1G`2TGXDut-RfAy9 z`|{v4KStU~iXiKa84uI+#kU{mj`8_ZvVSH!$CBcS*D2Bnb}b*XNE_jgDOGw^CZfV0 z!Yzp|yx$BdGF*alda>rPrg^8xY+#)4@fyzA4P6%Rd0^bn4G6X{p)GJM5R5fyWuSh> zd#HK|CY-TQW{G6GN7m=_z2Wo0+U4bXi{-Md(na4{FB=rB>l5b^NiXV7U2vv~{LkeN zcj`5*!|#{Y&NlFOEjvA7c_PDLKemXN%Eefr& z>>C=dU#ULevKV(1c3~C~apGDv-eqpQ5j@>c5flH+JgmI;wYl-mKBB1RSQ|e!Pjr%9 zYsX`U-j6GT`&NC^^QK&Xswq~{o!jew@yi|QF=!s9Iu(xG2506IsshkRZdIYfUkA@} z&*0Qhbe)`DkuaVxuWM+DPbJUJrI^;lq0fh~i9eE2G|_=3``ezu$r^605P&Nc?S)uA zaCKRh8_vA-K6}Kt%Q(Gkyxc0fJ3BF~Ol34je~@bzck$p39GMS8zQnhIu6DZ=23~uT z+ZVrmtTu+R4UY-U$j5=_7~S)M1!CU|v5~(;Ypf=M$Ii5%O_5gd%L31GDuMxAo#5c= zH1Q0)vNgUaLa|CPCg2O6Cb3ou7%+8lwk1yhSG5)0Nw7bVA+12 z+)B!yU8${d|7AElAU4*e5~)=BvME?8Wx50N0t-JjT#uNjT09H8@ityPhfRnh&*;j) zn3zmD{qO}|5~W_2y-?hxGE4w?ND$Zs$C819d&iH2kaY2IIy`q!oClvO{lfjWzbR-2%lfFqso6$ODxm}MlyWGY z#oq{Tm`Si5AQ+MR~ypmSPl#V7R3RtmnsMyw-GR*3| zD4L=$xC!}4bcNxVgtyS%0%z7hoSb(VgilfNbq?4os_&_Zj_*XNFQvq{oSl%4q00qk6%gdIwcQ}!&~mVSGYD-z^ldZec(w?g6^mGW zp8ui%Mlj-$G7oc5M(i& zVdjSSVXWSD_4R0cZZQ3ak;R7e4#p=*hJ~tOQEzAW-tPaZE)D|Cq0L3wr*QAr>kc%k zvM!M>qJ~hE`-E{n^SigsLBS7Y-oGWWq*rHW83`Lh8g|e<+_M=(L)w)M69j5ao~V^L zk&Ja#sylLCP3oh7oYNhS7~4FwmA>}>+nM5m{tN% ztFWbjIzV=1wISkQ3_DdjlDjXPjujH?t-j?x8_}!`6u&Ub?8NP4v~{SW*Gr7B@Gg>=)m(X%^k)a}m)uNPVa-;#5mLLhD&1J|cK;%sXV+)t zW(VD)uEPYY)Rp=s_F($2iW84zKmT{@D~#~{asJRdGpS=O;?&S`tQj zyq+p^#^^6)m1Turx?~@p|EQ&PsI;se;{h@94A`;TEUvrddXK6#Y^etRhnn6ULh%lgqDAF>_1s0Gpko~G>k zAu1kd1U$t ztjc}D4xe>)nd)J7TTo$$+2${`;#V|&Q!k!+={jZdyO9uy39F~ooRq9@TAB5WES=xH zsqDG9pB^>&Q)f7MJlI>WihX3n_TWL3p3~9^@mQzzZBtF}Ft%b_p)B61721{F9$ zapA}sQ}lghJJqN|PJZO`rAa&&64>TiJsV)OV)#y_0;7o>?i?n;nWMg`iK66RqV!g~*{QzjbhaU7-EShVmu`b870?bq!iE=hU0plOUsA8r|a z5Q}gcKdCN3dw<%0hTS>*xybRLt&weXf`72=+4q!Rv@xr5hb;$kM@kMulSzmb`Orel zFaaZn+j+d9cW5#Mlw{nf2)c%x=ImmEN>@PUJ!DpOf$y%gcIgeqz`(WQL^l_0SWHX%B;69*Ghu znhJC4$n;-7vd@BsOw^-k-gQx5Wns~I4#9cdN-QehXtGam5iF~m*K~AaF#2h$`%b%b z>io*LI||Z(^PcWmuVv2k(z%+6k@F{h_5#>)6rwGlR!tfmmr|r#81B1I6={B!?7W48 z@nIsi0`moBv7LrK(^ot!drL*H?H z{C#n=MLQ2lcb8b;!v}1}@|=YkB6~OMSVs8v7g_4JbAAot1OSG9(#X?JvAls&k|Z*E z9OWvalk484q`x{6H7rvviBNgK5!ztB)nYgNb>R$79Yxo%=@mK56P9%iHU4Qf8<}R> z95*Ns%r5#^_W3O`e%e#Z3xZrqO~Pz<=UIO zQVgcdWu8U`;WeOD!U~H0ZI#>wPY@*kuu5c)ZML8Mwo0TzXF?Ax7xq>|6#P%E5|2}- zIk6hFO2R?op{F7r-1u=s(cxsvaI!+P`!K8)ei-9hd-(PQ;WiU9UMSnUSA$5|x18;k z9gIpN0JNDWd5AS9v;&^h623>nMlES739mFyt%<6CyvzOg@Q@@TyVTbW`fJj@Y_0IV z+W9CL_5sNtxZ7+{ip}Jdzmfl|e@co~(C=#eg^v&Ti7zV-`W2!^7eE`3#eovOxCiQ+ zVj+F37}F1@BLJ>6Qr4qUkXN2rsLe4=)m76>3M*?C7o#|~a{zsh5^2kI0uZksGvx3~ z8e2^OAmXq?(j@saBEheM83&fV-da8cz&5Arl~L*YqJcV26Al_PQ_55 zhaN1^=$7bX>|myK9}D}h`S?n*v)3mj<=u3B4em#rjUmTcSTv#tK%&~Yuuz?G@y}M< zQgUAUkq;jMt*RzhD_@jDooUJCeh#-9PRSz4wl z@2EJ|J!qgQ;gP>ok%w(00MHdIA3AqQvJU(jh-2YlRj@(GmCUfM9dE|DYyK}ZZ}occ z5!I0ftmd5vC!Y~H>VH%v<>q}D{0YG>w9;#2rHO73>nXCJ9dOJCZD0ecnxb6fXdJA@f)+T#f?MWZ zsQU*)&xF=@jY}dE_|NNEI5K*x=dYEt&=!JLQx*btJxJQ|%XmKo?9 zaJG7$p0d_xZ{K*OY~eTkatLY2LsH<0rW4hX}9287J^KgY;Ytsw}ql*}3t$SG|uco76F! zJ`p-k`o*Vc_Uegom71}M$03b{Y+(u=+%VMs!h}DDC4fiMalRy=FVnVTOlU%9&$pI7HLr-mmqdy}g%eTZ((mDrTp&(({6)N7fl* z(O$9{qTy%BTyG}Y_{!O#{&r-YhNNN1hD_^lMpc$*53dRjze|DC`zR{UkY0+u3k$gqtNjE}b?5=G?I_#CltrJuG0snTiVf0BKT9M2KEDpz*>YS3KHbX8l) zK-4X~!bp@{xGz#6(WairyX-^nUOq5Hw`8$CoSsy23yWm@(j@veo`3Ak z?zuWdLu2xV2d+c*j3pvTTy(Z>eCJz=ctJ0Nw7I_%h!W5{IQo?IL&&!&deKB z>gC-w=snnZau$rDY77tCS1;lAlDX-C;mJdnA62XeHD8&|tuv1Y8uk{!I$p8cx?&cIJfj)u;pam+bS z*i`CiZ5;1Sw;w&oVL1hGKBt{qpTB@EOc2TujJGcsUrNvqe|IR{*RJIPa1~rIaKO4O z&Y8Al;?Kq3$Yj4>RXMl85H3FW(2bl~e#KUyFMw31v06$trDnCrMD=}&WM<8$I%Odf znMeA43+3w>h(!c<7y>Ev(4xD5`0LQKP##>=l~wr$GVHhQgPU>HGaXz0R=jnNEK?{Me+r1gJv3=5Ox-| zH`bk{OfwNH9^W6F5BFE8rV*%Em114g)`imRD1M8PL$7X`VSHy@;I|#H*BYW5U}!%4 zQ1^nAuALMfrZzGJpCAC>E``&{>l3*+wyGsvi4mtnIm(C6MPJ{&1O5d)nYzPpSco`9Atr-OOZD6GH#n zabE6Rwcd?Mvz2N%!gPNK0e*bBDh)>b1J9GU)lU$}$|!J8oyGy&j|W#O^8E!IKhH15 zlgVjt1E2GxWb+5S;!rslQ~jfby>q;DztF|7;MdE7`jzXGS@|D%ihf*NmU0^uv(~Ft zm^#8Bz*C^bmkSK4hRZzcVN4if)Row6ZoZs}jcsVQ)yt;5*=bElBGK2#7dHdPIeCPI zFd7K9;Qna3P9Bu`ES1Xfv-#cY7qS=S*T2V--t$R2D8BhkdTzk2K-t0#qRr5CqqEmx zTEo=7+~e?>n?#fG%fimlx4Q0yC2=Q995OM1nH=XS<|PCmj!mQnXUny z)F)rbJS7?YUIRO8#Vs0DR@biwhWeUEJWbT&XExSyHTvgqtHt*+9P{$xc>`oW&x}`_ zN~NkEQm-n_D2LTKRo_?IWZu7)>BnrJ5$WecirzzJi%q-m-L4c8cQ``}2R5s9UZB}q ze=9F_64NiUsD@)6 z96P8w-{O`UDb{d_9j9MX_bn^c*4B2`PT9^@hdomQ^RD|2_?O4`IzsuAEWT!(_+Kr~ zyD>3rx;L#o_!89KUH9L#Cn8MzcF~F0&Ayi0F^eNty+;$H9W&!Vs>Xto&7tE66Ecn0 zn3S_Hn&-aOWbd3eOzo8!&!&?5uos}IyuMom9=G_;Drlj&4(v&>=I794iD}Zqfghuj z^XBNX`#ay4FZY{1P?`@YrCaeSiLMNbwQfSg5`!t~9($fV8V>cF%}>CyoMj->G6N57 z!GEkVAgjzq>3z#WHWkkkJktMu>i}=R8lS!7DA&spy6|pL{pB}@s|`&lmmg-WSY3A0u%r70a zm`ZBtKdi)E2TQ~`mSQzE2*CY#qvXh3JM?qbcZWk!V-I+V6%Sq?H6J!gRBiv{;rmn> ze8X7lmip7^Qi;xcJLCA_kbM5LC)uGaWyrHCodxn}?c7BPjv4;RV5>}WgU()>MhRTx z)E=~c~foNHgO-hu%?(u2OsSN-R}$zhUu$8zLZvvVIJh$^J+E~J;zl? za#hF8{cKdLLn|Mwb`nlwI`3L09&cf>^pRzJ9<>;2G7q&f- zE+LP^Wi^n?PuZMh-hZdp@ppJRQCOSgrG3>XQm*c!!023UgVXw3IS)naf&z!hANJK< zyIe8-2=<%cHxhZh$NG9N+EpRJDUv_8jSl18;wKG#zxn*sInwDrh=`g~s-EG%S)dWy zO6ZqxtO%~KCa7tHCw^jT?-zJXXye0)Q*GGBKV3OoRp5b1;fLpzS}Y!Ys>ay!O}U#q)69fr7MQ*Nl$gk81$T6UIYJJR!5E7)y~{V^@v z?X`Jo1YUpOnDIa7R>qSwII;rVlM*?0DZ@$b9wITvDYfv=3ZUc^2iacm${n+6n7bRO zL;Zt_Za=rH#nUu75%Z}oI(4H%`uRqr%Ec=F{S(#UYWL#A!`P93TAjb2(XbgCyg~m` z^+{sRD*lOq{j_}~+P7T+55t-+8xpOXbdek+T(r|MxTn`oo@&$YaZWp+C|lT#@yAH*o8MVU!l&>mRRIPi|mU@D#{n*|0H;IrdyR zmaa!Rv#@-@i{tyufQd)7Sm#??s~L?~-R{NS$mh2%LvJQW+&y>H6G*oX zanxcqdnjC-2>cZZlc2%eTR2mKd5PXBTZ%SZ+`Z5H5#^nc#aTs zQ+_8SNvn}swx84Nwye-v2;ls_OzNY9%abvag2P6!6rrB0$0@PtLYbNwie&6Sa;2OW zw@ajaZANw9@vPg)rJ-xy>_e=v4I44k)-yKK8WP(^=cODsH=;cUBVb*K*cde|Q>Eau zWp}Ol48N}1`(Z3CgD_k1{HSt28RL`>{DJm<s5#;{SLeV{dpDS z{QMS{D6#uam)?D#{j^E`78G{-1iu380>R@A+#K~1%f7kWRel8G>?P-aAF0O;*jEPB zUX1KHBeorK>ULKNil#jR?KsqeZg4BdIiEs zXPZy(78VoDza1(&baCW++p6Z9BpnE!AshLIp*n9D7k7eMz~e=4-JXq#l6FuBN~qxn zT!R$xg8;<}!Wf&qdsZdEtkTfS5)(yp@PxK+?lSr+xIOiBP{|LP%J?m8m;ziW9qNY@cOqh8x8Zi0QSqYqQ`CF8UYoZ>;+rpgHY9McuXwp8%iAABj4g`ANFDp+ zj<4cu?7Mrrojv0$ASK>KsEw+-5oUD$tU8$MC3JLGa$UJAYEby|b+Y)4f`ryh856;k zU%OeSzK<4;*3L50>en@4*+VbK=qA!6i~e`&c2v1ZSn8uO3D-GGXj1b>fA;0r6OnxU zH%_;!2#kBoS(02@`rmN9R6>i@p&LF!MFoZmUk{zb?Cb3n$MxfN3bDsAcuVZe zos`IgwZ(RuA6y~d`@SVvcHBWYsx&GZIuIpVu@`9y$<3Og$%gS}aZt!*oa-AS_8PO^ z%gz$PkLjZwLn&=$ZaQm8M(9+fq5&T3*>TH78oRJC6fHHTdD&H%P5pYlAdmZX;fD?N z_pQF%D>OCJm_GhbjmKQV%waqRcfxrH_v6^1wuz1j0hMnda9?NjCz`$h~(Hv##W7wDVmS3up|05{g*B34?-_nPbn>5=|G-pP%0_BgjOG%gwu2Szq_Q$C@=9>wcE^b9}H%@854m$uMK3#-Z z8IOAKT5P0nW79@g`=(Cuox|t#!&(%dVpq(GXB0Q}u_g0h={m0~WnYy0wiVLam>#?H z9Uz$;<8kyWODznp!CU-ey4Az4+={YabV)j1OuCpa!liJ#Q`n$e9%cN}zH-QTO#g-P z6|V&UEs7v(X6+|WKI_Nf1JQe1;gE80>=DRiM*s5Af*YjeLTO$`)`NL`6sX~ciaeU* z?sgUb>&A|+6rVdJ+hXT0VJ(I|aaMZHD>XfyE>vJ;qr^(s&D>adxeO(W4p7a_B?O20^ z3Ed=CMI|#><^2ejLvp9$rsRX%=Be@>Z!K>|`L9!tKyr|V4qHZ;^bUVVMkQ(z>6SzX zjKSwt660*5e6s@n>|%5(kXHkipu%3C z+SIibB}S1Dr4Thp)DdXs3owpA_bL#7t=cifjzbTcB9^8YPZQgwqZqFt&Ja1q-HZv$ z8m(uSzBIkIO&#BfWPAR|6Kg~6o8HaiS)3`1$Mksa|f635BS!*C5e@|?X54BTwyD*tlK^yCfuuX;5#TwK`+1D)ql{II4IPnJ zx1wqn$ooNYJi(>PlEY5~EdGe^;=^FXnfAeg?U)jR#d;+NZ`YlCCz;_ zd2R^`UzHw%cA+(2Oy<2qpM>iW8IdpdYd`1V8yv^RW-r(3=cV*Lic=Jv_!7#TUtG^D zSC!XWF}~>RNK2@4>ILHx!Q6~1a69@pfCT4=LWtCn#B-bpsdbX>a?cd>1D%wz()zx- zs!0wEqQ_w1{ELwum=mWKyrPoFN=}-$5Ye3jL+N}m@7Wm<@oYuYB zYrHaMgdx0vL5xQHLtT-$)AOF%h>aw5r(*q@T&2>4^THs}jk+d;w_j&Jho9=LIv zLasamrVskhVc{X?R(Sd5%x!JqubQC&ibX8v3lxpN*(FX~w>$4IQ+7=%K;&9X0u02m zhYA-y4i6#im`dZ5E70foP=t( zb{-Up>ssJw?GUS_f^X~9t zJQ*;#^-PKQuG20`o~Oq8oj_;Y@mJWPUqxoHU-NNW(T)r}zq8 zw@i5No5vHP=wG@AzSFbv2hr8!IAk)9_%uEzOkM>GI!CJJaLZ{LbdRe9u8qf>rC@Ywh+EOSbq+2D^(+$Q4R2+yFR(Nv z!@W;gwt#eWPcV&G6@7e;G;bSjom`c(n9;aIjeGT6cycI7b7_a33*%ZNzoCBvy9!UL zl6zvc9*t*?yy9UzuL^(Q zd2Udmh2r_&qsYEOeyx~f{gX*9zmtTjN zkI0IZ);C1`)hKf?xtYmhDPqF?K~;`)EmZXwgcoA1A52xJy(DdhKPPLlkJr0p7bm<` zx@;Zyp~SA0$aJU5@pipJS0vbhwRB`C*Kh zFiXDGkfHQz>@Awdy>hKG*QYB_a@$8*7V3&>n5I?HE8v&$9%m4Z=(82XzC^cNQO)qW z&M4DwsjdRakCo+RdRj}wO2oE}0$tW!gD_%2;>bFINoKN_%y zziq^MUP~_n=7V{+=^NaC4w%o#5|NBupOAg^{p)%qk2KkP`GK85KEHfe)__$;1c4&M zO|iDoxY+)K|HXGjq67s@dc6#N>nGW>;so zg}ldlB9G6iCho@~?f`!=9`O7a+#JAl()Jmt<*RfAYW^Rg?tE#Y`#OL=l$_)? zz4PBg_+QPjsFRdAsygMuT>8fa=tutR?xUwmM{ox&uWLht)t}MlJsQKq9rv_d4GMqA zzG%?PH@<0KYNflgM$*E*lL!Wo=LAYt41vYx-CqnA%iI-pD=7-uw0f9sr8sCH@| z-_@7{s!XR2hEN?56?1F3!bo<{zl*K=V|q{Cui8CJGd>fMa7a8>82KN>T`)l0-TqJF zu91De);nP5Ui|ER%+A1B=2<2_uyQJ~NZn`^mn?JGk_7}b7y3+{zb2CBbrLv8wPRcJ zY37eG0{_p?&;Tw`-z&MO-e2aQ#O4&A*9zR#J`t(_9b|CQ1c0sI;r3z$apsM!tI|g0 zzD=5bMb)My_+U*+kTc}&RjWmzbn}Gq_Wp$ELHF6NK+JH!;0^sM=bA>>OV8^Fu|dPl zjvT`|V}4WHmaKYS)wpM6B_dZ|cRd$5)+OQ%wg$T(|E}GoxG3huX*TL)8|zf zvker~?29>dz-%rWcbfnB2b%^guD=m;jCOY5>_r3gU*&bjB_!&A0i*JM8HU||RszF8DG_U+7QnbI_ombrc)BMyx6PIo^Gp9mr;o0VB1y4p#5uB6T zh2Ln35XB}dzv7+4h89`*=7Wp0WdkZN>y6r63=E0tuLwXAA9@^|XK*1W^ahZ-7NhV) zB8p)Qp2O1L`1aE;c-cKBaz6<@T$00Z&uotU%2C9YK}pkBZM{+J9$Eg>If;X;MGQq3 zaU2QcICBgJKzYRq6XR$nfTm9Fqz?K&iS#A9Y-TNTe6;}(&xN}%oEVLfet?E9GjF|( zB%x1KZtlFrae~6N)2#Z`VOmc{F(7s0?sQlmyr*)Eaf#wWNXu&~LK3W^6&nI@%yzwuin#H@$GjD`Kwscbar+wkDxe0*|@-wAJR??pcPD>qvo(?riYN{+HeUANiWM<7fxAnJnTj98%<7>f-#8b5eP1^N5(3=$`~f_#RG ztJQ7Bytx6i(OPP;i)UWRcvYQNSP>{eb$US6`ZMd&8_A+XWjlKk&Pnp#!kI*)rC?>P zXfb*!&93NWZEk*mk6(zh*H2dH*PNuU3vVt@m2_S8YUkh~C-GQ&aFU4mQpU!n!O|N$ z71$n&+uq^P`ZoKU2w%YQ#^?oXfY5Zn{MMLeNQ16=OmMVgSs6TVvs#i|3-WakP#IR( z_;ziiP;ILlpq6hJ-lm9%UkxbH7DaQ9SDQLqj6d&{5FT?udMkKp!ATW7+jK>JdDy`?*d`(s_!4&%eX}GTuy2`~j2Cnf9GY=jQ^3_9Aw+3~XK4>8EPF7Ve2RdU((4z7p0T&@9R_EN$m{(+CqT#0%A@QBrJnxiYhBs_x+ z=+Ltv-GLAzZWNL0GJiv%c@J(oQ{NKrBtBU<@j6WS>mBv6+cXSoY*|}g4|*g3T857A z+N>XoOpNO0&QR05?Gp>GG8H#21cdKgj&Hkq!)~jTTV2DR7?cEC&SL zvWeqKz}VM=@f#e?DcAEN;fhYYskzo7ajYn%@2^<6Q5+r)R_E|N*bzefquN$qg{xTh>fqm{tG782H*`TvK1azi)PCI z@QFp=4LhxIdiC`DxzqD3OKneF$MddPIKdhQbFP)X`INanNb+dnh|%PsY!IjPDY1B| zBpl{W)w%HGirPEs%fh!We~jm2^8U~g!Gi-6{~}2Mib(8&5CN_Dw9Cd3Xpe<43=nZ9 zWdH*T-otJn%N%*WBO0+49P4dVeODhuZwdhmze&>AHAL*cDwHW~x$xC}VU5c?5<3?_ae9i2*>jgBuDe7IG>^6Z@mkEy?|T@u zauq7!Cf86$*8KB1Xtg=zeR)j+0-C>PMdRjY+y{bf2HEVEFrCK_B1V%2%qa1c{Ugv5 z!JB}@blPfN_r1CmuK&!`Uwhd2qtc-tnEL3xLDVz7&vso}7c*mO_&$bJjtYDeuxcz_ zLko=5TcB2KjW#OUf8^S`HAF~Qe&+p95@V2+By+4i2shr;p_J07u3YBtTI`uDj{$>uaew2JcsV5t*K5~KRt}19p*%6Y zrL_6*76bbA(+!a`Yybga)_Cr8ljszJEhB7}m#7;944}xhX$rgNCf=@b~AfW&Ba5PL?go z5`wW#%Lz2-5P}6DJdQo4c;p*;4;$SYkL7;N1?m>P#-94j`{DZEB~P%V)3}v65!GgN zY70014i3g`Xqnd&XA>9u-Q9BTpxCxM2lb|q{tKGRV0kgn#Sa_}p*qqR&03Q$ZcN5 z&+Vu4B*KMRKCCH;lqAZxve%11!j*pm1wh-@q}kM*cnuzkVSQ1{o~m`pF)K^2UF!0q zlcTJaD>hLU&7sFs+L3plJ8{>W2eKT$xoT@yOGxJV<`fN$?p9fb2$@^@a@prcT!NGRq&Ii43e z70F1x>xg9++YwMe_H)ps1Z)xktUaG`0T5du{sb*nC~uVIzVLZ@fg=~yRA>15)lg07 z13&4<&$hd!FehUF_Ho$KLs9LR2{6N?$_S7MNV^n zivL9XV~}1W5EyYPUHr}OJp`Y$+6%XP<@vqFYFRrM$n&IA@r3>e@Clj;o|;rY><4)# zBv4cNRaM=r$@fo4wR(8xGH}g%i`@1CL<>M)Akkdpa$WQRs-3}2&jGv0@lvG8Hk3T4 zt6LGiTYRoTV^N;@wbBtbjv_LfXIQu(ieCv%U*?5dK40I>Yi`ut5GE0Y4^OXT9Lf?f z9R1r;3x8RvUDa|fxXiB(RrOHq>n;_H$7;_YR}?f;fVCv;tlx|N5%SA5#X7b^y4vkT zQ3czOOmOOeyQsup^Q{U&uwnRyZQ6+Chk4n9G$;G?aG5*3W^d98t9S;zqYD)Gk}7^+ zs;lFqEgrU8d6vA@I7{#fUo*3tzUz0~ezLPnOOh?_jY4Vq8_p)~8i(Qw;+fG!fq`y$ zlP1Vuk%-xeOE!YlkM8q-jsMMb>uSikDVafg>|6CG_zVG8Rus(SVZ8Zomlv&rF7CV3 zmJerjhklR(Vrf@#*Ka_DK@^!Y?z}GbE@j}fXyX-Sc> zwmEyq_!M4v{F$D5t#IT5&TOfJ{`8ea6a#V=hT-@GF&g$fo)b862#vJi6zj z2zAl8{(4&7>31=HU8sQ|+A2K-^CE*6isAQ#OiV5rmm0BP@iUlc-W6U*MQV-Q<3`pFSwLcM2=rMh)6BYP(Q-s>%#&h8``E-@{EU0Qm)ggH{AO{@dpDuYaBY;W1O2 zrZW5_(WGNWt>uzS>_UA6_zbiXak&gC#U-KaZ+NA>^g`T&*87p(G3Q_JuvyK2r}%xR zm=glx)tz$n)gZ7^ioSnoZTU+#TWu4WlP7fqmc#%Lr^LoSNumSE2FRe&sg1THhhm4= zRtHvthG6ew>FtZsSuz|S5W12zhb**SqDOSSb`fF#%fBmV$uLX<-yc~Y`*Q&-#(Lvh zwaI>T7$WJwD!Tk-H!GicQi4%efc=0abrLDMT5aiRVVf{IIcsaUtI-UEzoqb9(Yg4k z;O0C#ie?H;`fI&LFxz}wCSMs-JauItBL^n2p*#m5!BH6;=nKoXn(ipY_h%FfzKrak zXR*wychAQ+WA1BSrZ}iWaYo2y_hYI>pNQhf2wkm(D%wdRl7+TS|Ae>%W6TkZigQRD zchSFx>!q`&Cv8&KY(&i5o#E}d+Q+KXpLz{E_H9GtGzY$we2gfuD1N1$Bbhgzu4H%w zI&P&2QEKTOCwtjG(pO$S0(BP6dA!>J()bcoI1T)A(M{L9K2jGJIV<(Jr&EKXhur#g z2yVyn$TCj-rAXjVp}ThpU>^IwQ!OV>(yEh0g&qQlmTc$Juh3)9MV!DAa9N5R=@NF% z@;xriLmLy{jJPl^r+9HQr}Cp4qkmC0Oonpc(=@*i~LN- zFV2%Y3*Xgikj~FR+T832dw@w;4z7~zdWmNv=i3D@fK1W9S~#fl?v1qk^RKDba$u|p zMnTAR65iwd3!q36ulc@uXfzX-r@D%EC?x80Z!K!SX??{c*zvkUqaczWXppq)U}2y+ zzVBN6eB`REI}|$9HMt?~p;LS%`ydc2*HBbtFosUg3xjh|*8GS&c-hxDw%D<&Z5#Q3JIXh#qc8G%-Wfwe|NNlG7o_VO&IkcZDKR6ad^`%Uq7 zOimV@qgIRUampcNt}m4kF9hqyMMnD~&#pS&bIj?Te?DeLzM@|wP}Fx}-}zo;z+-s! zF3;xJXRbIm0i%<=IU9l5v%f6eC)KV(CDHll$|^PWsbRz&V1{iVpI#nRcTNeC($ z(emsRD8{ArjpWmxL*KIaMjlPY*cYby2_EKUj*9Q8rKU;!k*W}Kv$dT{vdcq?T5gdC z$Fhiz9K|AAINL&USAiR9@D6nzfogV4XC-2A=OBNQ^gL?2Q3LF}ah(BD-Rp7}%NCXRQETNIZ$;fs`ve@?7;l2D>Yc8g+RiG3G) z^yz1uS(5&(mo=BwzM2GWM(8IPXU|50pKwpzn%Q$!T6C(`lKUGt55W*YLqRkkd42dK^P-0)B_`gCuHRJ?ycI;nQ~tc#^N}$2;{Y zLZzz5$)s8VhkLyNVeX)?{EZR7-#cs6K4x$Pa!;itYD|aKON|-3|2YC>V|vn~Dcyf4 zua7{wPPlj~0Vtzl2N&P~4U<(u)TNW20Id+#wuLDXf47z(6m}DaWeD=HORJ1J2b?$I z=Y!VBGUeuss0_}cdabLdHen8$IWNa6))fzrbEP{czUj~ZTI!wJUZUm+uJvhxVG;!J zYsBT=@Ag1BS=CZy2oqy;jc9g>Eei>8bB|V31dg69tJRm^02_lm5M7?U2DHXMPqZTh zgsPtbnqN@Xv?(v+L%6|{L7zStvVM-!**@Nr6QDWMxYwLQiT}3+8@mYR*kSp-m@psm z%>~)aju1~Oqp4qgwzWKWSRRt94R`+c72Wk|#FTZkX+ZH~?@=8EE47D?t^?ETwrA$7 zWn0?$Hza=@e}k+CU2M})4)tM?x~-cIK=*trKpSE0vk@VYsjg^g&M>Q77|o^!xW8X*t3T;j8!$)t=Jpa$n)cZAioDW4iQ{Jl+E z;kk4V{pvjz?I&NHO(-sSw-`%~qFg7bH42@m8ry{XWzrON(b2*i8U}h--*)SY1|EBE zR+r(93&I;AkQ7W)%6`1MS^(aJhcPmH8a;%n$EXVL-mVHXE|uuNR1l*Y-XwSf*2`xS zNvSQ3AxSkWk9zl~%(%T!40c2Pe7bY^(8xT%e{jb32o&V9wbUpai#e<4g9jj8%S(w} z1d6({@H?S_CWW8F*{+M9MK))L@+cG z#S5sytAgOo!9HH6@?0CV;uMaZh!3LwL@iI}CPwgnx>RNwP2)TmleHgpxjKsJ0cf<{ zvp)aj2>JKu^#6SR%QD41?a33NBqu0O!zcZPY|hMVgD>p^p(k168lM&q-Vr(mIF8b& zfFy8`pXkCkKN$ensj?V+YGj)uP^L1jpC(bh#gKOdP%n@t;3TmiV(?*(+QjjuL-3nK zQ%QF#z2S}Z6N}fU*fM9%2Rigt`FkbC<(8;Ji2OK@Rzc@_-8pQeqX0>^aJi&nF)!O@ z)}Gs#=Rw*1O^aYQoIlDsGgO`F|N1@KL*{iXRUKwxQ%(|oi@pG}(-|!k(02=T&a|GI zzBd;-*e!5vc@`J$IH(QscDzYseyu6GBJ~}vMO;iJ#44y*`c;g|;2T~)zw@W`^pfsF z+)dEm&J!`)&~Bh3HpV&QP!s_;ftrH>EAt%B*6u^*P-Ke)q8q%cL*oOmKQa)}+odPV z(^$0zeC;6Y%RzqVwCumlm1AW8fo!FL#`hIZ Date: Fri, 22 Oct 2021 17:43:05 +0200 Subject: [PATCH 097/156] switch test function to workflowv2 --- main/lib/idds/tests/core_tests.py | 18 +++++----- .../idds/tests/migrating_requests_v1_to_v2.py | 4 +-- main/lib/idds/tests/test_activelearning.py | 12 +++---- main/lib/idds/tests/test_atlaspandawork.py | 4 +-- main/lib/idds/tests/test_datacarousel.py | 8 ++--- main/lib/idds/tests/test_domapanda.py | 35 ++++++++++++++----- main/lib/idds/tests/test_hyperparameteropt.py | 4 +-- main/lib/idds/tests/test_running_data.py | 10 +++--- main/lib/idds/tests/test_workflow.py | 4 +-- monitor/conf.js | 12 +++---- 10 files changed, 66 insertions(+), 45 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 288cce41..f4634428 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,8 +102,8 @@ def show_works(req): print(work_ids) -""" -reqs = get_requests(request_id=256, with_detail=False, with_metadata=True) + +reqs = get_requests(request_id=198, with_detail=True, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) @@ -112,21 +112,23 @@ def show_works(req): pass sys.exit(0) -""" + +""" # reqs = get_requests() # print(len(reqs)) -""" for req in reqs: if req['request_id'] == 113: # print(req) # print(req['request_metadata']['workflow'].to_dict()) # print(json_dumps(req, sort_keys=True, indent=4)) pass -""" + +sys.exit(0) """ -tfs = get_transforms(request_id=301) + +tfs = get_transforms(request_id=194) for tf in tfs: # print(tf) # print(tf['transform_metadata']['work'].to_dict()) @@ -134,7 +136,7 @@ def show_works(req): pass sys.exit(0) - +""" msgs = retrieve_messages(workload_id=25972557) number_contents = 0 @@ -154,7 +156,7 @@ def show_works(req): sys.exit(0) """ -prs = get_processings(request_id=301) +prs = get_processings(request_id=194) i = 0 for pr in prs: # if pr['request_id'] == 91: diff --git a/main/lib/idds/tests/migrating_requests_v1_to_v2.py b/main/lib/idds/tests/migrating_requests_v1_to_v2.py index 67ac86e6..ef13d15e 100644 --- a/main/lib/idds/tests/migrating_requests_v1_to_v2.py +++ b/main/lib/idds/tests/migrating_requests_v1_to_v2.py @@ -11,9 +11,9 @@ from idds.common.constants import RequestType from idds.common.utils import get_rest_host from idds.core.requests import get_requests -from idds.workflow.workflow import Workflow +from idds.workflowv2.workflow import Workflow -from idds.atlas.workflow.atlasstageinwork import ATLASStageinWork +from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork from idds.client.clientmanager import ClientManager diff --git a/main/lib/idds/tests/test_activelearning.py b/main/lib/idds/tests/test_activelearning.py index 4844f1e0..8ef1b0af 100644 --- a/main/lib/idds/tests/test_activelearning.py +++ b/main/lib/idds/tests/test_activelearning.py @@ -32,12 +32,12 @@ # from idds.tests.common import get_example_real_tape_stagein_request # from idds.tests.common import get_example_prodsys2_tape_stagein_request -# from idds.workflow.work import Work, Parameter, WorkStatus -from idds.workflow.workflow import Condition, Workflow -# from idds.workflow.workflow import Workflow -# from idds.atlas.workflow.atlasstageinwork import ATLASStageinWork -from idds.atlas.workflow.atlaspandawork import ATLASPandaWork -from idds.atlas.workflow.atlasactuatorwork import ATLASActuatorWork +# from idds.workflowv2.work import Work, Parameter, WorkStatus +from idds.workflowv2.workflow import Condition, Workflow +# from idds.workflowv2.workflow import Workflow +# from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork +from idds.atlas.workflowv2.atlaspandawork import ATLASPandaWork +from idds.atlas.workflowv2.atlasactuatorwork import ATLASActuatorWork def get_task_id(output, error): diff --git a/main/lib/idds/tests/test_atlaspandawork.py b/main/lib/idds/tests/test_atlaspandawork.py index 1b5b33e9..f295795c 100644 --- a/main/lib/idds/tests/test_atlaspandawork.py +++ b/main/lib/idds/tests/test_atlaspandawork.py @@ -19,8 +19,8 @@ def get_workflow(): - from idds.workflow.workflow import Workflow, Condition - from idds.atlas.workflow.atlaspandawork import ATLASPandaWork + from idds.workflowv2.workflow import Workflow, Condition + from idds.atlas.workflowv2.atlaspandawork import ATLASPandaWork task_parameters1 = {"architecture": "", "cliParams": "prun --exec 'echo %RNDM:10 > seed.txt' --outputs seed.txt --nJobs 2 --outDS user.tmaeno.389eb4c5-5db6-4b80-82aa-9edfae6dfb38_000_top", diff --git a/main/lib/idds/tests/test_datacarousel.py b/main/lib/idds/tests/test_datacarousel.py index 3b3354a0..32ee936b 100644 --- a/main/lib/idds/tests/test_datacarousel.py +++ b/main/lib/idds/tests/test_datacarousel.py @@ -25,10 +25,10 @@ # from idds.tests.common import get_example_real_tape_stagein_request # from idds.tests.common import get_example_prodsys2_tape_stagein_request -# from idds.workflow.work import Work, Parameter, WorkStatus -# from idds.workflow.workflow import Condition, Workflow -from idds.workflow.workflow import Workflow -from idds.atlas.workflow.atlasstageinwork import ATLASStageinWork +# from idds.workflowv2.work import Work, Parameter, WorkStatus +# from idds.workflowv2.workflow import Condition, Workflow +from idds.workflowv2.workflow import Workflow +from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork def get_rucio_client(): diff --git a/main/lib/idds/tests/test_domapanda.py b/main/lib/idds/tests/test_domapanda.py index b42d19b7..70ec5a73 100644 --- a/main/lib/idds/tests/test_domapanda.py +++ b/main/lib/idds/tests/test_domapanda.py @@ -29,11 +29,11 @@ # from idds.tests.common import get_example_real_tape_stagein_request # from idds.tests.common import get_example_prodsys2_tape_stagein_request -# from idds.workflow.work import Work, Parameter, WorkStatus -# from idds.workflow.workflow import Condition, Workflow -from idds.workflow.workflow import Workflow -# from idds.atlas.workflow.atlasstageinwork import ATLASStageinWork -from idds.doma.workflow.domapandawork import DomaPanDAWork +# from idds.workflowv2.work import Work, Parameter, WorkStatus +# from idds.workflowv2.workflow import Condition, Workflow +from idds.workflowv2.workflow import Workflow +# from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork +from idds.doma.workflowv2.domapandawork import DomaPanDAWork task_queue = 'DOMA_LSST_GOOGLE_TEST' @@ -123,19 +123,38 @@ def setup_workflow(): output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#1'}], log_collections=[], dependency_map=taskN1.dependencies, task_name=taskN1.name, task_queue=task_queue, - task_cloud='US') + encode_command_line=True, + task_log={"dataset": "PandaJob_#{pandaid}/", + "destination": "local", + "param_type": "log", + "token": "local", + "type": "template", + "value": "log.tgz"}, + task_cloud='LSST') work2 = DomaPanDAWork(executable='echo', primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#2'}, output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#2'}], log_collections=[], dependency_map=taskN2.dependencies, task_name=taskN2.name, task_queue=task_queue, - task_cloud='US') + task_log={"dataset": "PandaJob_#{pandaid}/", + "destination": "local", + "param_type": "log", + "token": "local", + "type": "template", + "value": "log.tgz"}, + task_cloud='LSST') work3 = DomaPanDAWork(executable='echo', primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#3'}, output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#3'}], log_collections=[], dependency_map=taskN3.dependencies, task_name=taskN3.name, task_queue=task_queue, - task_cloud='US') + task_log={"dataset": "PandaJob_#{pandaid}/", + "destination": "local", + "param_type": "log", + "token": "local", + "type": "template", + "value": "log.tgz"}, + task_cloud='LSST') pending_time = 0.5 # pending_time = None diff --git a/main/lib/idds/tests/test_hyperparameteropt.py b/main/lib/idds/tests/test_hyperparameteropt.py index da0f6cd8..31d940ce 100644 --- a/main/lib/idds/tests/test_hyperparameteropt.py +++ b/main/lib/idds/tests/test_hyperparameteropt.py @@ -19,8 +19,8 @@ def get_workflow(): - from idds.workflow.workflow import Workflow - from idds.atlas.workflow.atlashpowork import ATLASHPOWork + from idds.workflowv2.workflow import Workflow + from idds.atlas.workflowv2.atlashpowork import ATLASHPOWork # request_metadata for predefined method 'nevergrad' request_metadata = {'workload_id': '20525135', 'sandbox': None, 'method': 'nevergrad', 'opt_space': {"A": {"type": "Choice", "params": {"choices": [1, 4]}}, "B": {"type": "Scalar", "bounds": [0, 5]}}, 'initial_points': [({'A': 1, 'B': 2}, 0.3), ({'A': 1, 'B': 3}, None)], 'max_points': 20, 'num_points_per_generation': 10} # noqa E501 diff --git a/main/lib/idds/tests/test_running_data.py b/main/lib/idds/tests/test_running_data.py index a3d22e89..3edec619 100644 --- a/main/lib/idds/tests/test_running_data.py +++ b/main/lib/idds/tests/test_running_data.py @@ -38,11 +38,11 @@ # from idds.tests.common import get_example_real_tape_stagein_request # from idds.tests.common import get_example_prodsys2_tape_stagein_request -# from idds.workflow.work import Work, Parameter, WorkStatus -# from idds.workflow.workflow import Condition, Workflow -from idds.workflow.workflow import Workflow -# from idds.atlas.workflow.atlasstageinwork import ATLASStageinWork -from idds.doma.workflow.domapandawork import DomaPanDAWork +# from idds.workflowv2.work import Work, Parameter, WorkStatus +# from idds.workflowv2.workflow import Condition, Workflow +from idds.workflowv2.workflow import Workflow +# from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork +from idds.doma.workflowv2.domapandawork import DomaPanDAWork task_queue = 'DOMA_LSST_GOOGLE_TEST' diff --git a/main/lib/idds/tests/test_workflow.py b/main/lib/idds/tests/test_workflow.py index e4b52fdf..dd83a6c2 100644 --- a/main/lib/idds/tests/test_workflow.py +++ b/main/lib/idds/tests/test_workflow.py @@ -21,8 +21,8 @@ from idds.common.constants import RequestType, RequestStatus from idds.common.utils import get_rest_host -from idds.workflow.work import Work # Parameter, WorkStatus -from idds.workflow.workflow import Condition, Workflow +from idds.workflowv2.work import Work # Parameter, WorkStatus +from idds.workflowv2.workflow import Condition, Workflow setup_logging(__name__) diff --git a/monitor/conf.js b/monitor/conf.js index 410cd4a1..5994e771 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus740.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus740.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus740.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus753.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus753.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus753.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus753.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus753.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus753.cern.ch:443/idds/monitor/null/null/false/false/true" } From d46206e3be5c81f014f65a52d8ae4841c3596d91 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:43:41 +0200 Subject: [PATCH 098/156] add doma workflowv2 --- doma/lib/idds/doma/workflowv2/__init__.py | 9 + .../lib/idds/doma/workflowv2/domapandawork.py | 953 ++++++++++++++++++ 2 files changed, 962 insertions(+) create mode 100644 doma/lib/idds/doma/workflowv2/__init__.py create mode 100644 doma/lib/idds/doma/workflowv2/domapandawork.py diff --git a/doma/lib/idds/doma/workflowv2/__init__.py b/doma/lib/idds/doma/workflowv2/__init__.py new file mode 100644 index 00000000..1bb8ee59 --- /dev/null +++ b/doma/lib/idds/doma/workflowv2/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py new file mode 100644 index 00000000..9cfe7495 --- /dev/null +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -0,0 +1,953 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 +# - Sergey Padolski, , 2020 + + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + +import datetime +import os +import traceback + +from idds.common import exceptions +from idds.common.constants import (TransformType, CollectionStatus, CollectionType, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.workflowv2.work import Work, Processing +from idds.workflowv2.workflow import Condition + + +class DomaCondition(Condition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None): + super(DomaCondition, self).__init__(cond=cond, current_work=current_work, + true_work=true_work, false_work=false_work) + + +class DomaPanDAWork(Work): + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, + work_tag='lsst', exec_type='panda', sandbox=None, work_id=None, + primary_input_collection=None, other_input_collections=None, + input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + logger=None, dependency_map=None, task_name="", + task_queue=None, processing_type=None, + prodSourceLabel='test', task_type='test', + maxwalltime=90000, maxattempt=5, core_count=1, + encode_command_line=False, + num_retries=5, + task_log=None, + task_cloud=None, + task_rss=0): + + super(DomaPanDAWork, self).__init__(executable=executable, arguments=arguments, + parameters=parameters, setup=setup, work_type=TransformType.Processing, + work_tag=work_tag, exec_type=exec_type, sandbox=sandbox, work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + input_collections=input_collections, + output_collections=output_collections, + log_collections=log_collections, + release_inputs_after_submitting=True, + logger=logger) + # self.pandamonitor = None + self.panda_url = None + self.panda_url_ssl = None + self.panda_monitor = None + + self.dependency_map = dependency_map + self.dependency_map_deleted = [] + # self.logger.setLevel(logging.DEBUG) + + self.task_name = task_name + self.set_work_name(task_name) + self.queue = task_queue + self.dep_tasks_id_names_map = {} + self.executable = executable + self.processingType = processing_type + self.prodSourceLabel = prodSourceLabel + self.task_type = task_type + self.maxWalltime = maxwalltime + self.maxAttempt = maxattempt + self.core_count = core_count + self.task_log = task_log + + self.encode_command_line = encode_command_line + self.task_cloud = task_cloud + self.task_rss = task_rss + + self.retry_number = 0 + self.num_retries = num_retries + + self.load_panda_urls() + + def my_condition(self): + if self.is_finished(): + return True + return False + + def load_panda_config(self): + panda_config = ConfigParser.SafeConfigParser() + if os.environ.get('IDDS_PANDA_CONFIG', None): + configfile = os.environ['IDDS_PANDA_CONFIG'] + if panda_config.read(configfile) == [configfile]: + return panda_config + + configfiles = ['%s/etc/panda/panda.cfg' % os.environ.get('IDDS_HOME', ''), + '/etc/panda/panda.cfg', '/opt/idds/etc/panda/panda.cfg', + '%s/etc/panda/panda.cfg' % os.environ.get('VIRTUAL_ENV', '')] + for configfile in configfiles: + if panda_config.read(configfile) == [configfile]: + return panda_config + return panda_config + + def load_panda_urls(self): + panda_config = self.load_panda_config() + # self.logger.debug("panda config: %s" % panda_config) + self.panda_url = None + self.panda_url_ssl = None + self.panda_monitor = None + + if panda_config.has_section('panda'): + if panda_config.has_option('panda', 'panda_monitor_url'): + self.panda_monitor = panda_config.get('panda', 'panda_monitor_url') + os.environ['PANDA_MONITOR_URL'] = self.panda_monitor + # self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) + if panda_config.has_option('panda', 'panda_url'): + self.panda_url = panda_config.get('panda', 'panda_url') + os.environ['PANDA_URL'] = self.panda_url + # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if panda_config.has_option('panda', 'panda_url_ssl'): + self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') + os.environ['PANDA_URL_SSL'] = self.panda_url_ssl + # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + + if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']: + self.panda_monitor = os.environ['PANDA_MONITOR_URL'] + # self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) + if not self.panda_url and 'PANDA_URL' in os.environ and os.environ['PANDA_URL']: + self.panda_url = os.environ['PANDA_URL'] + # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: + self.panda_url_ssl = os.environ['PANDA_URL_SSL'] + # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + + def set_agent_attributes(self, attrs, req_attributes=None): + if 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: + attrs['life_time'] = None + super(DomaPanDAWork, self).set_agent_attributes(attrs) + if 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: + self.num_retries = int(self.agent_attributes['num_retries']) + + def depend_on(self, work): + for job in self.dependency_map: + inputs_dependency = job["dependencies"] + + for input_d in inputs_dependency: + task_name = input_d['task'] + if task_name == work.task_name: + return True + return False + + def poll_external_collection(self, coll): + try: + if coll.status in [CollectionStatus.Closed]: + return coll + else: + coll.coll_metadata['bytes'] = 1 + coll.coll_metadata['availability'] = 1 + coll.coll_metadata['events'] = 1 + coll.coll_metadata['is_open'] = True + coll.coll_metadata['run_number'] = 1 + coll.coll_metadata['did_type'] = 'DATASET' + coll.coll_metadata['list_all_files'] = False + + # if (not self.dependency_map_deleted and not self.dependency_map): + if not self.has_new_inputs: + coll.coll_metadata['is_open'] = False + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + coll.coll_metadata['coll_type'] = CollectionType.Dataset + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_input_collections(self): + """ + *** Function called by Transformer agent. + """ + colls = [self._primary_input_collection] + self._other_input_collections + for coll_int_id in colls: + coll = self.collections[coll_int_id] + # if self.is_internal_collection(coll): + # coll = self.poll_internal_collection(coll) + # else: + # coll = self.poll_external_collection(coll) + coll = self.poll_external_collection(coll) + self.collections[coll_int_id] = coll + return super(DomaPanDAWork, self).get_input_collections() + + def get_mapped_inputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + inputs = mapped_input_output_maps[map_id]['inputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_input = inputs[0] + for ip in inputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_input = ip + ret.append(primary_input) + return ret + + def get_mapped_outputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + outputs = mapped_input_output_maps[map_id]['outputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_output = outputs[0] + for ip in outputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_output = ip + ret.append(primary_output) + return ret + + def map_file_to_content(self, coll_id, scope, name): + content = {'coll_id': coll_id, + 'scope': scope, + 'name': name, # or a different file name from the dataset name + 'bytes': 1, + 'adler32': '12345678', + 'min_id': 0, + 'max_id': 1, + 'content_type': ContentType.File, + # 'content_relation_type': content_relation_type, + # here events is all events for eventservice, not used here. + 'content_metadata': {'events': 1}} + return content + + def is_all_dependency_tasks_available(self, inputs_dependency, task_name_to_coll_map): + for input_d in inputs_dependency: + task_name = input_d['task'] + if (task_name not in task_name_to_coll_map # noqa: W503 + or 'outputs' not in task_name_to_coll_map[task_name] # noqa: W503 + or not task_name_to_coll_map[task_name]['outputs']): # noqa: W503 + return False + return True + + def get_unmapped_jobs(self, mapped_input_output_maps={}): + mapped_outputs = self.get_mapped_outputs(mapped_input_output_maps) + mapped_outputs_name = [ip['name'] for ip in mapped_outputs] + unmapped_jobs = [] + for job in self.dependency_map: + output_name = job['name'] + if output_name not in mapped_outputs_name: + unmapped_jobs.append(job) + return unmapped_jobs + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + *** Function called by Transformer agent. + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + new_input_output_maps = {} + + unmapped_jobs = self.get_unmapped_jobs(mapped_input_output_maps) + if not unmapped_jobs: + self.set_has_new_inputs(False) + return new_input_output_maps + + if unmapped_jobs: + mapped_keys = mapped_input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + + input_coll = self.get_input_collections()[0] + input_coll_id = input_coll.coll_id + output_coll = self.get_output_collections()[0] + output_coll_id = output_coll.coll_id + + task_name_to_coll_map = self.get_work_name_to_coll_map() + + for job in unmapped_jobs: + output_name = job['name'] + inputs_dependency = job["dependencies"] + + if self.is_all_dependency_tasks_available(inputs_dependency, task_name_to_coll_map): + input_content = self.map_file_to_content(input_coll_id, input_coll.scope, output_name) + output_content = self.map_file_to_content(output_coll_id, output_coll.scope, output_name) + new_input_output_maps[next_key] = {'inputs_dependency': [], + 'logs': [], + 'inputs': [input_content], + 'outputs': [output_content]} + for input_d in inputs_dependency: + task_name = input_d['task'] + input_name = input_d['inputname'] + input_d_coll = task_name_to_coll_map[task_name]['outputs'][0] + input_d_content = self.map_file_to_content(input_d_coll['coll_id'], input_d_coll['scope'], input_name) + new_input_output_maps[next_key]['inputs_dependency'].append(input_d_content) + + # all inputs are parsed. move it to dependency_map_deleted + # self.dependency_map_deleted.append(job) + next_key += 1 + else: + # not all inputs for this job can be parsed. + # self.dependency_map.append(job) + pass + + # self.logger.debug("get_new_input_output_maps, new_input_output_maps: %s" % str(new_input_output_maps)) + self.logger.debug("get_new_input_output_maps, new_input_output_maps len: %s" % len(new_input_output_maps)) + return new_input_output_maps + + def use_dependency_to_release_jobs(self): + """ + *** Function called by Transformer agent. + """ + return True + + def get_processing(self, input_output_maps=[], without_creating=False): + """ + *** Function called by Transformer agent. + + If there is already an active processing for this work, will do nothing. + If there is no active processings, create_processing will be called. + """ + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + if not without_creating: + # return None + return self.create_processing(input_output_maps) + return None + + def create_processing(self, input_output_maps=[]): + """ + *** Function called by Transformer agent. + + :param input_output_maps: new maps from inputs to outputs. + """ + # avoid duplicated task name + self.task_name = self.task_name + "_" + str(self.get_work_id()) + + in_files = [] + for job in self.dependency_map: + in_files.append(job['name']) + + task_param_map = {} + task_param_map['vo'] = 'wlcg' + if self.queue and len(self.queue) > 0: + task_param_map['site'] = self.queue + task_param_map['workingGroup'] = 'lsst' + task_param_map['nFilesPerJob'] = 1 + task_param_map['nFiles'] = len(in_files) + task_param_map['noInput'] = True + task_param_map['pfnList'] = in_files + task_param_map['taskName'] = self.task_name + task_param_map['userName'] = 'iDDS' + task_param_map['taskPriority'] = 900 + task_param_map['architecture'] = '' + task_param_map['transUses'] = '' + task_param_map['transHome'] = None + if self.encode_command_line: + # task_param_map['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c-enc' + task_param_map['transPath'] = 'https://storage.googleapis.com/drp-us-central1-containers/bash-c-enc' + task_param_map['encJobParams'] = True + else: + # task_param_map['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c' + task_param_map['transPath'] = 'https://storage.googleapis.com/drp-us-central1-containers/bash-c' + task_param_map['processingType'] = self.processingType + task_param_map['prodSourceLabel'] = self.prodSourceLabel + task_param_map['taskType'] = self.task_type + task_param_map['coreCount'] = self.core_count + task_param_map['skipScout'] = True + task_param_map['cloud'] = self.task_cloud + if self.task_rss and self.task_rss > 0: + task_param_map['ramCount'] = self.task_rss + task_param_map['ramUnit'] = 'MB' + + task_param_map['inputPreStaging'] = True + task_param_map['prestagingRuleID'] = 123 + task_param_map['nChunksToWait'] = 1 + task_param_map['maxCpuCount'] = self.maxWalltime + task_param_map['maxWalltime'] = self.maxWalltime + task_param_map['maxFailure'] = self.maxAttempt + task_param_map['maxAttempt'] = self.maxAttempt + task_param_map['log'] = self.task_log + task_param_map['jobParameters'] = [ + {'type': 'constant', + 'value': self.executable, # noqa: E501 + }, + ] + + processing_metadata = {'task_param': task_param_map} + proc = Processing(processing_metadata=processing_metadata) + proc.workload_id = None + self.add_processing_to_processings(proc) + self.active_processings.append(proc.internal_id) + return proc + + def submit_panda_task(self, processing): + try: + from pandatools import Client + + proc = processing['processing_metadata']['processing'] + task_param = proc.processing_metadata['task_param'] + return_code = Client.insertTaskParams(task_param, verbose=True) + if return_code[0] == 0: + return return_code[1][1] + else: + self.logger.warn("submit_panda_task, return_code: %s" % str(return_code)) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc())) + return None + + def submit_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + proc = processing['processing_metadata']['processing'] + if proc.workload_id: + # if 'task_id' in processing['processing_metadata'] and processing['processing_metadata']['task_id']: + pass + else: + task_id = self.submit_panda_task(processing) + # processing['processing_metadata']['task_id'] = task_id + # processing['processing_metadata']['workload_id'] = task_id + proc.workload_id = task_id + if task_id: + proc.submitted_at = datetime.datetime.utcnow() + + def get_panda_task_id(self, processing): + from pandatools import Client + + start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=10) + start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + status, results = Client.getJobIDsJediTasksInTimeRange(start_time, task_type=self.task_type, verbose=False) + if status != 0: + self.logger.warn("Error to poll latest tasks in last ten hours: %s, %s" % (status, results)) + return None + + proc = processing['processing_metadata']['processing'] + task_id = None + for req_id in results: + task_name = results[req_id]['taskName'] + if proc.workload_id is None and task_name == self.task_name: + task_id = results[req_id]['jediTaskID'] + # processing['processing_metadata']['task_id'] = task_id + # processing['processing_metadata']['workload_id'] = task_id + proc.workload_id = task_id + if task_id: + proc.submitted_at = datetime.datetime.utcnow() + + return task_id + + def poll_panda_task_status(self, processing): + if 'processing' in processing['processing_metadata']: + from pandatools import Client + + proc = processing['processing_metadata']['processing'] + status, task_status = Client.getTaskStatus(proc.workload_id) + if status == 0: + return task_status + else: + return 'failed' + return None + + def get_processing_status_from_panda_status(self, task_status): + if task_status in ['registered', 'defined', 'assigning']: + processing_status = ProcessingStatus.Submitting + elif task_status in ['ready', 'pending', 'scouting', 'scouted', 'prepared', 'topreprocess', 'preprocessing']: + processing_status = ProcessingStatus.Submitted + elif task_status in ['running', 'toretry', 'toincexec', 'throttled']: + processing_status = ProcessingStatus.Running + elif task_status in ['done']: + processing_status = ProcessingStatus.Finished + elif task_status in ['finished', 'paused']: + # finished, finishing, waiting it to be done + processing_status = ProcessingStatus.SubFinished + elif task_status in ['failed', 'aborted', 'broken', 'exhausted']: + # aborting, tobroken + processing_status = ProcessingStatus.Failed + else: + # finished, finishing, aborting, topreprocess, preprocessing, tobroken + # toretry, toincexec, rerefine, paused, throttled, passed + processing_status = ProcessingStatus.Submitted + return processing_status + + def is_all_contents_terminated_and_with_missing(self, input_output_maps): + with_missing = False + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if not content['status'] in [ContentStatus.Failed, ContentStatus.FinalFailed, + ContentStatus.Lost, ContentStatus.Deleted, + ContentStatus.Missing]: + return False + if not with_missing and content['status'] in [ContentStatus.Missing]: + with_missing = True + if with_missing: + return True + return False + + def reactive_contents(self, input_output_maps): + updated_contents = [] + for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] if 'inputs' in input_output_maps[map_id] else [] + outputs = input_output_maps[map_id]['outputs'] if 'outputs' in input_output_maps[map_id] else [] + inputs_dependency = input_output_maps[map_id]['inputs_dependency'] if 'inputs_dependency' in input_output_maps[map_id] else [] + + all_outputs_available = True + for content in outputs: + if not content['status'] in [ContentStatus.Available]: + all_outputs_available = False + break + + if not all_outputs_available: + for content in inputs + outputs: + update_content = {'content_id': content['content_id'], + 'status': ContentStatus.New, + 'substatus': ContentStatus.New} + updated_contents.append(update_content) + for content in inputs_dependency: + if content['status'] not in [ContentStatus.Available]: + update_content = {'content_id': content['content_id'], + 'status': ContentStatus.New, + 'substatus': ContentStatus.New} + updated_contents.append(update_content) + return updated_contents + + def sort_panda_jobids(self, input_output_maps): + panda_job_ids = {} + panda_id_to_map_ids = {} + map_id_without_panda_ids = [] + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if content['status'] not in panda_job_ids: + panda_job_ids[content['status']] = [] + + if 'panda_id' in content['content_metadata']: + panda_job_ids[content['status']].append(content['content_metadata']['panda_id']) + panda_id_to_map_ids[content['content_metadata']['panda_id']] = map_id + else: + map_id_without_panda_ids.append(map_id) + + return panda_job_ids, map_id_without_panda_ids, panda_id_to_map_ids + + def get_registered_panda_jobids(self, input_output_maps): + panda_job_ids, map_id_without_panda_ids, panda_id_to_map_ids = self.sort_panda_jobids(input_output_maps) + unterminated_panda_ids = [] + finished_panda_ids = [] + failed_panda_ids = [] + for key in panda_job_ids: + if key in [ContentStatus.Available]: + finished_panda_ids += panda_job_ids[key] + elif key in [ContentStatus.Failed, ContentStatus.FinalFailed, + ContentStatus.Lost, ContentStatus.Deleted, + ContentStatus.Missing]: + failed_panda_ids += panda_job_ids[key] + else: + unterminated_panda_ids += panda_job_ids[key] + return finished_panda_ids + failed_panda_ids, unterminated_panda_ids, map_id_without_panda_ids, panda_id_to_map_ids + + def get_map_id_from_input(self, input_output_maps, input_file): + map_keys = list(input_output_maps.keys()) + map_keys.reverse() + for map_id in map_keys: + inputs = input_output_maps[map_id]['inputs'] + # outputs = input_output_maps[map_id]['outputs'] + for content in inputs: + if content['name'] == input_file: + return map_id + return None + + def get_content_status_from_panda_status(self, job_info): + jobstatus = job_info.jobStatus + if jobstatus in ['finished', 'merging']: + return ContentStatus.Available + elif jobstatus in ['failed', 'closed', 'cancelled', 'lost', 'broken', 'missing']: + attempt_nr = int(job_info.attemptNr) if job_info.attemptNr else 0 + max_attempt = int(job_info.maxAttempt) if job_info.maxAttempt else 0 + if (attempt_nr >= max_attempt) and (attempt_nr >= self.maxAttempt): + return ContentStatus.FinalFailed + else: + return ContentStatus.Failed + else: + return ContentStatus.Processing + + def get_update_contents_from_map_id(self, map_id, input_output_maps, job_info): + outputs = input_output_maps[map_id]['outputs'] + update_contents = [] + for content in outputs: + status = self.get_content_status_from_panda_status(job_info) + content['substatus'] = status + + if 'panda_id' in content['content_metadata'] and content['content_metadata']['panda_id']: + # if content['content_metadata']['panda_id'] != job_info.PandaID: + if content['content_metadata']['panda_id'] < job_info.PandaID: + # new panda id is the bigger one. + if 'old_panda_id' not in content['content_metadata']: + content['content_metadata']['old_panda_id'] = [] + if content['content_metadata']['panda_id'] not in content['content_metadata']['old_panda_id']: + content['content_metadata']['old_panda_id'].append(content['content_metadata']['panda_id']) + content['content_metadata']['panda_id'] = job_info.PandaID + + update_contents.append(content) + return update_contents + + def map_panda_ids(self, unregistered_job_ids, input_output_maps): + self.logger.debug("map_panda_ids, unregistered_job_ids: %s" % str(unregistered_job_ids)) + from pandatools import Client + + # updated_map_ids = [] + full_update_contents = [] + chunksize = 2000 + chunks = [unregistered_job_ids[i:i + chunksize] for i in range(0, len(unregistered_job_ids), chunksize)] + for chunk in chunks: + jobs_list = Client.getJobStatus(chunk, verbose=0)[1] + for job_info in jobs_list: + if job_info and job_info.Files and len(job_info.Files) > 0: + for job_file in job_info.Files: + # if job_file.type in ['log']: + if job_file.type not in ['pseudo_input']: + continue + if ':' in job_file.lfn: + pos = job_file.lfn.find(":") + input_file = job_file.lfn[pos + 1:] + # input_file = job_file.lfn.split(':')[1] + else: + input_file = job_file.lfn + map_id = self.get_map_id_from_input(input_output_maps, input_file) + if map_id: + update_contents = self.get_update_contents_from_map_id(map_id, input_output_maps, job_info) + full_update_contents += update_contents + return full_update_contents + + def get_status_changed_contents(self, unterminated_job_ids, input_output_maps, panda_id_to_map_ids): + self.logger.debug("get_status_changed_contents, unterminated_job_ids: %s" % str(unterminated_job_ids)) + from pandatools import Client + + full_update_contents = [] + chunksize = 2000 + chunks = [unterminated_job_ids[i:i + chunksize] for i in range(0, len(unterminated_job_ids), chunksize)] + for chunk in chunks: + jobs_list = Client.getJobStatus(chunk, verbose=0)[1] + for job_info in jobs_list: + panda_id = job_info.PandaID + map_id = panda_id_to_map_ids[panda_id] + update_contents = self.get_update_contents_from_map_id(map_id, input_output_maps, job_info) + full_update_contents += update_contents + return full_update_contents + + def get_final_update_contents(self, input_output_maps): + update_contents = [] + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] if 'outputs' in input_output_maps[map_id] else [] + for content in outputs: + if (content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed]): + content['content_metadata']['old_final_status'] = content['substatus'] + content['substatus'] = ContentStatus.FinalFailed + update_contents.append(content) + + return update_contents + + def poll_panda_task(self, processing=None, input_output_maps=None): + task_id = None + try: + from pandatools import Client + + jobs_ids = None + if processing: + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + if task_id is None: + task_id = self.get_panda_task_id(processing) + + if task_id: + # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) + self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) + if task_info[0] != 0: + self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) + return ProcessingStatus.Submitting, {} + + task_info = task_info[1] + + processing_status = self.get_processing_status_from_panda_status(task_info["status"]) + + if processing_status in [ProcessingStatus.SubFinished]: + if self.retry_number < self.num_retries: + self.reactivate_processing(processing) + processing_status = ProcessingStatus.Submitted + self.retry_number += 1 + + jobs_ids = task_info['PandaID'] + ret_get_registered_panda_jobids = self.get_registered_panda_jobids(input_output_maps) + terminated_job_ids, unterminated_job_ids, map_id_without_panda_ids, panda_id_to_map_ids = ret_get_registered_panda_jobids + + registered_job_ids = terminated_job_ids + unterminated_job_ids + unregistered_job_ids = [] + for job_id in jobs_ids: + if job_id not in registered_job_ids: + unregistered_job_ids.append(job_id) + + map_update_contents = self.map_panda_ids(unregistered_job_ids, input_output_maps) + status_changed_update_contents = self.get_status_changed_contents(unterminated_job_ids, input_output_maps, panda_id_to_map_ids) + final_update_contents = [] + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if (unregistered_job_ids or unterminated_job_ids): + # there are still polling contents, should not terminate the task. + log_warn = "Processing (%s) with panda id (%s) is %s, however there are still unregistered_job_ids(%s) or unterminated_job_ids(%s)" % (processing['processing_id'], + task_id, + processing_status, + str(unregistered_job_ids), + str(unterminated_job_ids)) + log_warn = log_warn + ". Keep the processing status as running now." + self.logger.warn(log_warn) + processing_status = ProcessingStatus.Running + else: + final_update_contents = self.get_final_update_contents(input_output_maps) + if final_update_contents: + processing_status = ProcessingStatus.Running + return processing_status, map_update_contents + status_changed_update_contents + final_update_contents + else: + return ProcessingStatus.Failed, {} + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + self.logger.error(msg) + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException(msg) + return ProcessingStatus.Submitting, [] + + def kill_processing(self, processing): + try: + if processing: + from pandatools import Client + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + # task_id = processing['processing_metadata']['task_id'] + # Client.killTask(task_id) + Client.finishTask(task_id, soft=False) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def kill_processing_force(self, processing): + try: + if processing: + from pandatools import Client + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + # task_id = processing['processing_metadata']['task_id'] + Client.killTask(task_id) + # Client.finishTask(task_id, soft=True) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def reactivate_processing(self, processing): + try: + if processing: + from pandatools import Client + # task_id = processing['processing_metadata']['task_id'] + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + + # Client.retryTask(task_id) + status, out = Client.retryTask(task_id, newParams={}) + self.logger.warn("Retry processing(%s) with task id(%s): %s, %s" % (processing['processing_id'], task_id, status, out)) + # Client.reactivateTask(task_id) + # Client.resumeTask(task_id) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def poll_processing_updates(self, processing, input_output_maps): + """ + *** Function called by Carrier agent. + """ + updated_contents = [] + update_processing = {} + reset_expired_at = False + reactive_contents = [] + # self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) + + if processing: + proc = processing['processing_metadata']['processing'] + if proc.tocancel: + self.logger.info("Cancelling processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tocancel = False + proc.polling_retries = 0 + elif proc.tosuspend: + self.logger.info("Suspending processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tosuspend = False + proc.polling_retries = 0 + elif proc.toresume: + self.logger.info("Resuming processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.reactivate_processing(processing) + reset_expired_at = True + proc.toresume = False + proc.polling_retries = 0 + proc.has_new_updates() + reactive_contents = self.reactive_contents(input_output_maps) + # elif self.is_processing_expired(processing): + elif proc.toexpire: + self.logger.info("Expiring processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.toexpire = False + proc.polling_retries = 0 + elif proc.tofinish or proc.toforcefinish: + self.logger.info("Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.tofinish = False + proc.toforcefinish = False + proc.polling_retries = 0 + elif self.is_all_contents_terminated_and_with_missing(input_output_maps): + self.logger.info("All contents terminated(There are Missing contents). Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + + processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) + self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) + self.logger.debug("poll_processing_updates, update_contents: %s" % str(poll_updated_contents)) + + if poll_updated_contents: + proc.has_new_updates() + for content in poll_updated_contents: + updated_content = {'content_id': content['content_id'], + 'substatus': content['substatus'], + 'content_metadata': content['content_metadata']} + updated_contents.append(updated_content) + + content_substatus = {'finished': 0, 'unfinished': 0} + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if content.get('substatus', ContentStatus.New) != ContentStatus.Available: + content_substatus['unfinished'] += 1 + else: + content_substatus['finished'] += 1 + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] and updated_contents: + self.logger.info("Processing %s is terminated, but there are still contents to be flushed. Waiting." % (proc.workload_id)) + # there are still polling contents, should not terminate the task. + processing_status = ProcessingStatus.Running + + if processing_status in [ProcessingStatus.SubFinished] and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + # found that a 'done' panda task has got a 'finished' status. Maybe in this case 'finished' is a transparent status. + if proc.polling_retries is None: + proc.polling_retries = 0 + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if proc.polling_retries is not None and proc.polling_retries < 3: + self.logger.info("processing %s polling_retries(%s) < 3, keep running" % (processing['processing_id'], proc.polling_retries)) + processing_status = ProcessingStatus.Running + proc.polling_retries += 1 + else: + proc.polling_retries = 0 + + if proc.in_operation_time(): + processing_status = ProcessingStatus.Running + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': processing_status}} + if reset_expired_at: + processing['expired_at'] = None + update_processing['parameters']['expired_at'] = None + proc.polling_retries = 0 + # if (processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] + # or processing['status'] in [ProcessingStatus.Resuming]): # noqa W503 + # using polling_retries to poll it again when panda may update the status in a delay(when issuing retryTask, panda will not update it without any delay). + update_processing['parameters']['status'] = ProcessingStatus.Resuming + proc.status = update_processing['parameters']['status'] + + self.logger.debug("poll_processing_updates, task: %s, update_processing: %s" % + (proc.workload_id, str(update_processing))) + self.logger.debug("poll_processing_updates, task: %s, updated_contents: %s" % + (proc.workload_id, str(updated_contents))) + self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % + (proc.workload_id, str(reactive_contents))) + return update_processing, updated_contents + reactive_contents, {} + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + for map_id in registered_input_output_maps: + outputs = registered_input_output_maps[map_id]['outputs'] + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + self.status_statistics = status_statistics + self.logger.debug("registered_input_output_maps, status_statistics: %s" % str(status_statistics)) + return status_statistics + + def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + super(DomaPanDAWork, self).syn_work_status(registered_input_output_maps, all_updates_flushed, output_statistics, to_release_input_contents) + # self.get_status_statistics(registered_input_output_maps) + self.status_statistics = output_statistics + + self.logger.debug("syn_work_status, self.active_processings: %s" % str(self.active_processings)) + self.logger.debug("syn_work_status, self.has_new_inputs(): %s" % str(self.has_new_inputs)) + self.logger.debug("syn_work_status, coll_metadata_is_open: %s" % + str(self.collections[self._primary_input_collection].coll_metadata['is_open'])) + self.logger.debug("syn_work_status, primary_input_collection_status: %s" % + str(self.collections[self._primary_input_collection].status)) + + self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) + self.logger.debug("syn_work_status(%s): is_input_collections_closed: %s" % (str(self.get_processing_ids()), str(self.is_input_collections_closed()))) + self.logger.debug("syn_work_status(%s): has_new_inputs: %s" % (str(self.get_processing_ids()), str(self.has_new_inputs))) + self.logger.debug("syn_work_status(%s): has_to_release_inputs: %s" % (str(self.get_processing_ids()), str(self.has_to_release_inputs()))) + self.logger.debug("syn_work_status(%s): to_release_input_contents: %s" % (str(self.get_processing_ids()), str(to_release_input_contents))) + + if self.is_processings_terminated() and self.is_input_collections_closed() and not self.has_new_inputs and not self.has_to_release_inputs() and not to_release_input_contents: + # if not self.is_all_outputs_flushed(registered_input_output_maps): + if not all_updates_flushed: + self.logger.warn("The work processings %s is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform" % str(self.get_processing_ids())) + return + + keys = self.status_statistics.keys() + if len(keys) == 1: + if ContentStatus.Available.name in keys: + self.status = WorkStatus.Finished + else: + self.status = WorkStatus.Failed + else: + self.status = WorkStatus.SubFinished + elif self.is_processings_running(): + self.status = WorkStatus.Running + else: + self.status = WorkStatus.Transforming + + if self.is_processings_started(): + self.started = True From 60d7ce31f0ca35af7252f371bf663c98bc38e310 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:44:04 +0200 Subject: [PATCH 099/156] add atlas workflowv2 --- atlas/lib/idds/atlas/workflowv2/__init__.py | 9 + .../atlas/workflowv2/atlasactuatorwork.py | 460 +++++++++++ .../idds/atlas/workflowv2/atlascondorwork.py | 241 ++++++ .../lib/idds/atlas/workflowv2/atlasdagwork.py | 498 ++++++++++++ .../lib/idds/atlas/workflowv2/atlashpowork.py | 751 ++++++++++++++++++ .../idds/atlas/workflowv2/atlaspandawork.py | 744 +++++++++++++++++ .../idds/atlas/workflowv2/atlasstageinwork.py | 440 ++++++++++ 7 files changed, 3143 insertions(+) create mode 100644 atlas/lib/idds/atlas/workflowv2/__init__.py create mode 100644 atlas/lib/idds/atlas/workflowv2/atlasactuatorwork.py create mode 100644 atlas/lib/idds/atlas/workflowv2/atlascondorwork.py create mode 100644 atlas/lib/idds/atlas/workflowv2/atlasdagwork.py create mode 100644 atlas/lib/idds/atlas/workflowv2/atlashpowork.py create mode 100644 atlas/lib/idds/atlas/workflowv2/atlaspandawork.py create mode 100644 atlas/lib/idds/atlas/workflowv2/atlasstageinwork.py diff --git a/atlas/lib/idds/atlas/workflowv2/__init__.py b/atlas/lib/idds/atlas/workflowv2/__init__.py new file mode 100644 index 00000000..865b774e --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2019 diff --git a/atlas/lib/idds/atlas/workflowv2/atlasactuatorwork.py b/atlas/lib/idds/atlas/workflowv2/atlasactuatorwork.py new file mode 100644 index 00000000..f7a83988 --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlasactuatorwork.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + +import copy +import json +import os +import traceback + +from rucio.client.client import Client as RucioClient +from rucio.common.exception import (CannotAuthenticate as RucioCannotAuthenticate) + +from idds.common import exceptions +from idds.common.constants import (TransformType, CollectionType, CollectionStatus, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.common.utils import run_command +# from idds.workflowv2.work import Work +from idds.workflowv2.work import Processing +from idds.atlas.workflowv2.atlascondorwork import ATLASCondorWork + + +class ATLASActuatorWork(ATLASCondorWork): + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, + work_tag='actuating', exec_type='local', sandbox=None, work_id=None, + name=None, + primary_input_collection=None, other_input_collections=None, input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + logger=None, + workload_id=None, + agent_attributes=None, + output_json=None): + """ + Init a work/task/transformation. + + :param setup: A string to setup the executable enviroment, it can be None. + :param executable: The executable. + :param arguments: The arguments. + :param parameters: A dict with arguments needed to be replaced. + :param work_type: The work type like data carousel, hyperparameteroptimization and so on. + :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. + :param sandbox: The sandbox. + :param work_id: The work/task id. + :param primary_input_collection: The primary input collection. + :param other_input_collections: List of the input collections. + :param output_collections: List of the output collections. + # :param workflow: The workflow the current work belongs to. + :param sandbox: The sandbox to be uploaded or the container path. + :param executable: The executable command. + :param arguments: The arguments for the executable. + """ + + super(ATLASActuatorWork, self).__init__(executable=executable, arguments=arguments, work_tag=work_tag, + parameters=parameters, setup=setup, work_type=TransformType.Actuating, + exec_type=exec_type, sandbox=sandbox, work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + input_collections=input_collections, + output_collections=output_collections, + log_collections=log_collections, + logger=logger, + agent_attributes=agent_attributes) + + self.output_json = output_json + + self.terminated = False + self.tocancel = False + + # if self.agent_attributes and 'atlashpowork' in self.agent_attributes: + # self.agent_attributes = self.agent_attributes['atlashpowork'] + # self.logger.info("agent_attributes: %s" % self.agent_attributes) + + # if self.agent_attributes and 'workdir' in self.agent_attributes and self.agent_attributes['workdir']: + # self.set_workdir(self.agent_attributes['workdir']) + # self.logger.info("workdir: %s" % self.get_workdir()) + + if agent_attributes: + self.set_agent_attributes(agent_attributes) + + def set_agent_attributes(self, attrs, req_attributes=None): + super(ATLASActuatorWork, self).set_agent_attributes(attrs) + + if self.agent_attributes and 'workdir' in self.agent_attributes and self.agent_attributes['workdir']: + if req_attributes and 'request_id' in req_attributes and 'workload_id' in req_attributes and 'transform_id' in req_attributes: + req_dir = 'request_%s_%s/transform_%s' % (req_attributes['request_id'], + req_attributes['workload_id'], + req_attributes['transform_id']) + self.set_workdir(os.path.join(self.agent_attributes['workdir'], req_dir)) + self.logger.info("workdir: %s" % self.get_workdir()) + + ########################################## # noqa E266 + def generate_new_task(self): + self.logger.info("Work %s parameters for next task: %s" % (self.internal_id, str(self.get_parameters_for_next_task()))) + if self.get_parameters_for_next_task(): + return True + else: + return False + + ####### functions for transformer ######## # noqa E266 + ###################################### # noqa E266 + + def set_output_data(self, data): + # overwrite to transfer the output of current task to next task + super(ATLASActuatorWork, self).set_output_data(data) + super(ATLASActuatorWork, self).set_parameters_for_next_task(data) + + def get_rucio_client(self): + try: + client = RucioClient() + except RucioCannotAuthenticate as error: + self.logger.error(error) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc())) + return client + + def poll_external_collection(self, coll): + try: + if coll.status in [CollectionStatus.Closed]: + return coll + else: + client = self.get_rucio_client() + did_meta = client.get_metadata(scope=coll.scope, name=coll.name) + coll.coll_metadata['bytes'] = did_meta['bytes'] + coll.coll_metadata['total_files'] = did_meta['length'] + coll.coll_metadata['availability'] = did_meta['availability'] + coll.coll_metadata['events'] = did_meta['events'] + coll.coll_metadata['is_open'] = did_meta['is_open'] + coll.coll_metadata['run_number'] = did_meta['run_number'] + coll.coll_metadata['did_type'] = did_meta['did_type'] + coll.coll_metadata['list_all_files'] = False + + if (('is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']) + or ('force_close' in coll.coll_metadata and coll.coll_metadata['force_close'])): # noqa: W503 + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + if 'did_type' in coll.coll_metadata: + if coll.coll_metadata['did_type'] == 'DATASET': + coll_type = CollectionType.Dataset + elif coll.coll_metadata['did_type'] == 'CONTAINER': + coll_type = CollectionType.Container + else: + coll_type = CollectionType.File + else: + coll_type = CollectionType.Dataset + coll.coll_metadata['coll_type'] = coll_type + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_input_collections(self): + # return [self.primary_input_collection] + self.other_input_collections + colls = [self._primary_input_collection] + self._other_input_collections + for coll_int_id in colls: + coll = self.collections[coll_int_id] + coll = self.poll_external_collection(coll) + self.collections[coll_int_id] = coll + return super(ATLASActuatorWork, self).get_input_collections() + + def get_input_contents(self): + """ + Get all input contents from DDM. + """ + try: + ret_files = [] + coll = self.collections[self._primary_input_collection] + ret_file = {'coll_id': coll['coll_id'], + 'scope': coll['scope'], + 'name': coll['name'], + 'bytes': coll.coll_metadata['bytes'], + 'adler32': None, + 'min_id': 0, + 'max_id': coll.coll_metadata['total_files'], + 'content_type': ContentType.File, + 'content_metadata': {'total_files': coll['coll_metadata']['total_files']} + } + ret_files.append(ret_file) + return ret_files + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_mapped_inputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + inputs = mapped_input_output_maps[map_id]['inputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_input = inputs[0] + for ip in inputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_input = ip + ret.append(primary_input) + return ret + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + inputs = self.get_input_contents() + mapped_inputs = self.get_mapped_inputs(mapped_input_output_maps) + mapped_inputs_scope_name = [ip['scope'] + ":" + ip['name'] for ip in mapped_inputs] + + new_inputs = [] + new_input_output_maps = {} + for ip in inputs: + ip_scope_name = ip['scope'] + ":" + ip['name'] + if ip_scope_name not in mapped_inputs_scope_name: + new_inputs.append(ip) + + # to avoid cheking new inputs if there are no new inputs anymore + if (not new_inputs and 'status' in self.collections[self._primary_input_collection] + and self.collections[self._primary_input_collection]['status'] in [CollectionStatus.Closed]): # noqa: W503 + self.set_has_new_inputs(False) + else: + mapped_keys = mapped_input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + for ip in new_inputs: + out_ip = copy.deepcopy(ip) + out_ip['coll_id'] = self.collections[self._primary_output_collection]['coll_id'] + new_input_output_maps[next_key] = {'inputs': [ip], + 'outputs': [out_ip]} + next_key += 1 + + self.unfinished_points = 1 + + return new_input_output_maps + + def get_processing(self, input_output_maps, without_creating=False): + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + if not without_creating: + return self.create_processing(input_output_maps) + return None + + def create_processing(self, input_output_maps): + processing_metadata = {} + proc = Processing(processing_metadata=processing_metadata) + self.add_processing_to_processings(proc) + self.active_processings.append(proc.internal_id) + return proc + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + + self.total_output_files = 0 + self.processed_output_file = 0 + + for map_id in registered_input_output_maps: + outputs = registered_input_output_maps[map_id]['outputs'] + + self.total_output_files += 1 + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + + if content['status'] == ContentStatus.Available: + self.processed_output_file += 1 + + self.status_statistics = status_statistics + return status_statistics + + def syn_collection_status(self): + input_collections = self.get_input_collections() + output_collections = self.get_output_collections() + # log_collections = self.get_log_collections() + + for input_collection in input_collections: + input_collection['total_files'] = 1 + input_collection['processed_files'] = 1 + + for output_collection in output_collections: + output_collection['total_files'] = self.total_output_files + output_collection['processed_files'] = self.processed_output_file + + def syn_work_status(self, registered_input_output_maps): + self.get_status_statistics(registered_input_output_maps) + + self.syn_collection_status() + + if self.is_processings_terminated() and not self.has_new_inputs: + if self.is_processings_finished(): + self.status = WorkStatus.Finished + elif self.is_processings_failed(): + self.status = WorkStatus.Failed + elif self.is_processings_subfinished(): + self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming + + ####### functions for carrier ######## # noqa E266 + ###################################### # noqa E266 + + def get_rucio_setup_env(self): + script = "export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase\n" + script += "source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh\n" + script += "export RUCIO_ACCOUNT=pilot\n" + script += "localSetupRucioClients\n" + return script + + def generate_processing_script_sandbox(self, processing): + arguments = self.parse_arguments() + + script = "#!/bin/bash\n\n" + script += self.get_rucio_setup_env() + script += "\n" + + script += "sandbox=%s\n" % str(self.sandbox) + script += "executable=%s\n" % str(self.executable) + script += "arguments=%s\n" % str(arguments) + script += "output_json=%s\n" % str(self.output_json) + script += "\n" + + script += "env\n" + script += "echo $X509_USER_PROXY\n" + script += "\n" + + script += "echo 'user id:'\n" + script += "id\n" + script += "\n" + + script += "wget $sandbox\n" + script += 'base_sandbox="$(basename -- $sandbox)"\n' + script += 'tar xzf $base_sandbox\n' + + dataset = self.collections[self._primary_input_collection] + script += 'rucio download %s:%s\n' % (dataset['scope'], dataset['name']) + script += 'chmod +x %s\n' % str(self.executable) + script += "echo '%s' '%s'\n" % (str(self.executable), str(arguments)) + script += '%s %s\n' % (str(self.executable), str(arguments)) + + script += 'ls\n\n' + + long_id = self.get_long_id(processing) + script_name = 'processing_%s.sh' % long_id + script_name = os.path.join(self.get_working_dir(processing), script_name) + with open(script_name, 'w') as f: + f.write(script) + run_command("chmod +x %s" % script_name) + return script_name + + def get_output_json(self, processing): + # job_dir = self.get_working_dir(processing) + if self.output_json: + return self.output_json + elif 'output_json' in self.agent_attributes and self.agent_attributes['output_json']: + output_json = self.agent_attributes['output_json'] + else: + output_json = 'idds_output.json' + return output_json + + def generate_processing_script(self, processing): + self.output_json = self.get_output_json(processing) + + script_name = self.generate_processing_script_sandbox(processing) + return script_name, None + + def get_output_files(self, processing): + return [self.output_json] + + def submit_processing(self, processing): + if 'job_id' in processing['processing_metadata']: + pass + else: + job_id, errors = self.submit_condor_processing(processing) + if errors: + self.add_errors(errors) + processing['processing_metadata']['job_id'] = job_id + processing['processing_metadata']['errors'] = str(self.get_errors()) + + def abort_processing(self, processing): + self.tocancel = True + + def parse_processing_outputs(self, processing): + request_id = processing['request_id'] + workload_id = processing['workload_id'] + processing_id = processing['processing_id'] + + if not self.output_json: + return None, 'Request(%s)_workload(%s)_processing(%s) output_json(%s) is not defined' % (request_id, workload_id, + processing_id, self.output_json) + + job_dir = self.get_working_dir(processing) + full_output_json = os.path.join(job_dir, self.output_json) + if not os.path.exists(full_output_json): + return None, '%s is not created' % str(full_output_json) + else: + try: + with open(full_output_json, 'r') as f: + data = f.read() + outputs = json.loads(data) + if not outputs: + return outputs, "No points generated: the outputs is empty" + return outputs, None + except Exception as ex: + return None, 'Failed to load the content of %s: %s' % (str(full_output_json), str(ex)) + + def poll_processing(self, processing): + job_status, job_err_msg = self.poll_condor_job_status(processing, processing['processing_metadata']['job_id']) + processing_outputs = None + if job_status in [ProcessingStatus.Finished]: + job_outputs, parser_errors = self.parse_processing_outputs(processing) + if job_outputs: + processing_status = ProcessingStatus.Finished + processing_err = None + processing_outputs = job_outputs + else: + processing_status = ProcessingStatus.Failed + processing_err = parser_errors + elif self.tocancel: + processing_status = ProcessingStatus.Cancelled + processing_outputs = None + processing_err = None + else: + processing_status = job_status + processing_err = job_err_msg + return processing_status, processing_outputs, processing_err + + def poll_processing_updates(self, processing, input_output_maps): + processing_status, processing_outputs, processing_err = self.poll_processing(processing) + + processing_metadata = processing['processing_metadata'] + if not processing_metadata: + processing_metadata = {} + if processing_err: + processing_err = processing_err.strip() + if processing_err: + self.add_errors(processing_err) + processing_metadata['errors'] = str(self.get_errors()) + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': processing_status, + 'processing_metadata': processing_metadata, + 'output_metadata': processing_outputs}} + + updated_contents = [] + return update_processing, updated_contents, {} diff --git a/atlas/lib/idds/atlas/workflowv2/atlascondorwork.py b/atlas/lib/idds/atlas/workflowv2/atlascondorwork.py new file mode 100644 index 00000000..15b97249 --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlascondorwork.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 + +import os + +from idds.common.constants import (ProcessingStatus) +from idds.common.utils import run_command +from idds.workflowv2.work import Work + + +class ATLASCondorWork(Work): + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, + work_type=None, work_tag='hpo', exec_type='local', sandbox=None, work_id=None, + primary_input_collection=None, other_input_collections=None, input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + agent_attributes=None, + logger=None): + """ + Init a work/task/transformation. + + :param setup: A string to setup the executable enviroment, it can be None. + :param executable: The executable. + :param arguments: The arguments. + :param parameters: A dict with arguments needed to be replaced. + :param work_type: The work type like data carousel, hyperparameteroptimization and so on. + :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. + :param sandbox: The sandbox. + :param work_id: The work/task id. + :param primary_input_collection: The primary input collection. + :param other_input_collections: List of the input collections. + :param output_collections: List of the output collections. + # :param workflow: The workflow the current work belongs to. + """ + super(ATLASCondorWork, self).__init__(executable=executable, arguments=arguments, + parameters=parameters, setup=setup, work_type=work_type, + exec_type=exec_type, sandbox=sandbox, work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + input_collections=input_collections, + output_collections=output_collections, + log_collections=log_collections, + agent_attributes=agent_attributes, + logger=logger) + + def get_long_id(self, processing): + request_id = processing['request_id'] + workload_id = processing['workload_id'] + processing_id = processing['processing_id'] + long_id = '%s_%s_%s' % (request_id, workload_id, processing_id) + return long_id + + def get_working_dir(self, processing): + # request_id = processing['request_id'] + # workload_id = processing['workload_id'] + processing_id = processing['processing_id'] + + job_dir = 'processing_%s' % (processing_id) + job_dir = os.path.join(self.get_workdir(), job_dir) + if not os.path.exists(job_dir): + os.makedirs(job_dir) + return job_dir + + def generate_processing_submit_file(self, processing): + script_name, err_msg = self.generate_processing_script(processing) + if not script_name: + return None, err_msg + + input_files = self.get_input_files(processing) + output_files = self.get_output_files(processing) + # self.logger.info("input_files: %s, output_files: %s" % (str(input_files), str(output_files))) + + long_id = self.get_long_id(processing) + + jdl = "#Agent jdl file\n" + jdl += "Universe = vanilla\n" + jdl += "Notification = Never\n" + jdl += "initialdir = %s\n" % self.get_working_dir(processing) + jdl += "Executable = %s\n" % script_name + # jdl += "Arguments = %s\na" % (self.get_job_dir(processing_id)) + jdl += "GetEnv = False\n" + jdl += "Output = " + 'processing_%s' % long_id + ".$(ClusterId).$(ProcId).out\n" + jdl += "Error = " + 'processing_%s' % long_id + ".$(ClusterId).$(ProcId).err\n" + jdl += "Log = " + 'processing_%s' % long_id + ".$(ClusterId).$(ProcId).log\n" + jdl += "stream_output = False\n" + jdl += "stream_error = False\n" + # jdl += 'Requirements = ((Arch == "X86_64") && (regexp("SLC",OpSysLongName)))\n' + # jdl += 'Requirements = ((Arch == "X86_64") && (regexp("CentOS",OpSysLongName)))\n' + # jdl += "transfer_input_files = file1, file2\n" + jdl += "should_transfer_files = yes\n" + + tf_inputs = [script_name] + if input_files: + tf_inputs = tf_inputs + input_files + tf_outputs = output_files + + # self.logger.info("tf_inputs: %s, tf_outputs: %s" % (str(tf_inputs), str(tf_outputs))) + + # if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + # proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + # tf_inputs = tf_inputs + [os.environ['X509_USER_PROXY']] + + if tf_inputs: + jdl += "transfer_input_files = %s\n" % (str(','.join(tf_inputs))) + if tf_outputs: + jdl += "transfer_output_files = %s\n" % (str(','.join(tf_outputs))) + + jdl += "WhenToTransferOutput = ON_EXIT_OR_EVICT\n" + jdl += "OnExitRemove = TRUE\n" + # jdl += '+JobFlavour = "espresso"\n' + # jdl += '+JobFlavour = "tomorrow"\n' + # jdl += '+JobFlavour = "testmatch"\n' + # jdl += '+JobFlavour = "nextweek"\n' + jdl += '+JobType="ActiveLearning"\n' + # jdl += '+AccountingGroup ="group_u_ATLASWISC.all"\n' + jdl += '+Processing_id = "%s"\n' % long_id + jdl += "RequestCpus = 1\n" + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + jdl += "x509userproxy = %s\n" % str(os.environ['X509_USER_PROXY']) + jdl += "Queue 1\n" + + submit_file = 'processing_%s.jdl' % long_id + submit_file = os.path.join(self.get_working_dir(processing), submit_file) + with open(submit_file, 'w') as f: + f.write(jdl) + return submit_file, None + + def get_input_files(self, processing): + return [] + + def get_output_files(self, processing): + return [] + + def submit_condor_processing(self, processing): + jdl_file, err_msg = self.generate_processing_submit_file(processing) + if not jdl_file: + return None, err_msg + + cmd = "condor_submit " + jdl_file + status, output, error = run_command(cmd) + jobid = None + self.logger.info("submiting the job to cluster: %s" % cmd) + self.logger.info("status: %s, output: %s, error: %s " % (status, output, error)) + if status == 0 or str(status) == '0': + if output and 'submitted to cluster' in output: + for line in output.split('\n'): + if 'submitted to cluster' in line: + jobid = line.split(' ')[-1].replace('.', '') + return jobid, None + return None, output + error + + def get_job_err_message(self, job_workdir, job_err): + try: + if not job_err: + return '' + if not job_err.startswith("/") and job_workdir: + job_err = os.path.join(job_workdir, job_err) + if not os.path.exists(job_err): + return '' + with open(job_err, "r") as myfile: + data = myfile.readlines() + data = str(data) + data = data[-1000:] + return data + except Exception as e: + self.logger.error("Failed to read job error file(workdir: %s, error file: %s): %s" % (job_workdir, job_err, e)) + return '' + + def poll_condor_job_status(self, processing, job_id): + # 0 Unexpanded U + # 1 Idle I + # 2 Running R + # 3 Removed X + # 4 Completed C + # 5 Held H + # 6 Submission_err E + cmd = "condor_q -format '%s' ClusterId -format ' %s' Processing_id -format ' %s' JobStatus -format ' %s' Iwd -format ' %s' Cmd -format ' %s' Err " + str(job_id) + status, output, error = run_command(cmd) + self.logger.info("poll job status: %s" % cmd) + self.logger.info("status: %s, output: %s, error: %s" % (status, output, error)) + if status == 0 and len(output) == 0: + cmd = "condor_history -format '%s' ClusterId -format ' %s' Processing_id -format ' %s' JobStatus -format ' %s' Iwd -format ' %s' Cmd -format ' %s' Err " + str(job_id) + status, output, error = run_command(cmd) + self.logger.info("poll job status: %s" % cmd) + self.logger.info("status: %s, output: %s, error: %s" % (status, output, error)) + + ret_err = '' + job_cmd_msg, job_err_msg = '', '' + if status == 0: + lines = output.split('\n') + for line in lines: + c_job_id, c_processing_id, c_job_status, job_workdir, job_cmd, job_err = line.split(' ') + if str(c_job_id) != str(job_id): + continue + + processing_id = self.get_long_id(processing) + c_job_status = int(c_job_status) + if c_processing_id != processing_id: + final_job_status = ProcessingStatus.Failed + ret_err = 'jobid and the processing_id mismatched' + else: + job_status = c_job_status + if job_status < 2: + final_job_status = ProcessingStatus.Submitted + elif job_status == 2: + final_job_status = ProcessingStatus.Submitted + elif job_status == 2: + final_job_status = ProcessingStatus.Running + elif job_status == 3: + final_job_status = ProcessingStatus.Cancelled + elif job_status == 4: + final_job_status = ProcessingStatus.Finished + else: + final_job_status = ProcessingStatus.Failed + + if final_job_status in [ProcessingStatus.Failed]: + job_cmd_msg = self.get_job_err_message(job_workdir, job_cmd) + job_cmd_msg = job_cmd_msg[-500:] + job_err_msg = self.get_job_err_message(job_workdir, job_err) + else: + final_job_status = ProcessingStatus.Submitted + + # if output: + # ret_err += output + if error: + ret_err += error + if job_cmd_msg: + ret_err += "Command output: " + job_cmd_msg + if job_err_msg: + ret_err += "Stderr: " + job_err_msg + + return final_job_status, ret_err diff --git a/atlas/lib/idds/atlas/workflowv2/atlasdagwork.py b/atlas/lib/idds/atlas/workflowv2/atlasdagwork.py new file mode 100644 index 00000000..a1b2c764 --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlasdagwork.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + +import copy +import json +import os +import traceback +import uuid +import urllib +import ssl + +from idds.common import exceptions +from idds.common.constants import (TransformType, CollectionType, CollectionStatus, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.workflowv2.work import Work +from idds.workflowv2.workflow import Condition +import logging + + +class DomaCondition(Condition): + def __init__(self, cond=None, current_work=None, true_work=None, false_work=None): + super(DomaCondition, self).__init__(cond=cond, current_work=current_work, + true_work=true_work, false_work=false_work) + + +class DomaLSSTWork(Work): + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, + work_tag='lsst', exec_type='panda', sandbox=None, work_id=None, + primary_input_collection=None, other_input_collections=None, input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + logger=None, dependency_map=None, task_name=""): + """ + Init a work/task/transformation. + + :param setup: A string to setup the executable enviroment, it can be None. + :param executable: The executable. + :param arguments: The arguments. + :param parameters: A dict with arguments needed to be replaced. + :param work_type: The work type like data carousel, hyperparameteroptimization and so on. + :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. + :param sandbox: The sandbox. + :param work_id: The work/task id. + :param primary_input_collection: The primary input collection. + :param other_input_collections: List of the input collections. + :param output_collections: List of the output collections. + # :param workflow: The workflow the current work belongs to. + """ + + super(DomaLSSTWork, self).__init__(executable=executable, arguments=arguments, + parameters=parameters, setup=setup, work_type=TransformType.Processing, + work_tag=work_tag, exec_type=exec_type, sandbox=sandbox, work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + input_collections=input_collections, + output_collections=output_collections, + log_collections=log_collections, + logger=logger) + self.pandamonitor = None + self.dependency_map = dependency_map + self.logger.setLevel(logging.DEBUG) + self.task_name = task_name + + def my_condition(self): + if self.is_finished(): + return True + return False + + def jobs_to_idd_ds_status(self, jobstatus): + if jobstatus == 'finished': + return ContentStatus.Available + elif jobstatus == 'failed ': + return ContentStatus.Failed + else: + return ContentStatus.Processing + + def load_panda_config(self): + panda_config = ConfigParser.SafeConfigParser() + if os.environ.get('IDDS_PANDA_CONFIG', None): + configfile = os.environ['IDDS_PANDA_CONFIG'] + if panda_config.read(configfile) == [configfile]: + return panda_config + + configfiles = ['%s/etc/panda/panda.cfg' % os.environ.get('IDDS_HOME', ''), + '/etc/panda/panda.cfg', '/opt/idds/etc/panda/panda.cfg', + '%s/etc/panda/panda.cfg' % os.environ.get('VIRTUAL_ENV', '')] + for configfile in configfiles: + if panda_config.read(configfile) == [configfile]: + return panda_config + return panda_config + + def load_panda_monitor(self): + panda_config = self.load_panda_config() + self.logger.info("panda config: %s" % panda_config) + if panda_config.has_section('panda'): + if panda_config.has_option('panda', 'pandamonitor'): + pandamonitor = panda_config.get('panda', 'pandamonitor') + return pandamonitor + return None + + def poll_external_collection(self, coll): + try: + # if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: + if 'status' in coll and coll['status'] in [CollectionStatus.Closed]: + return coll + else: + # client = self.get_rucio_client() + # did_meta = client.get_metadata(scope=coll['scope'], name=coll['name']) + if 'coll_metadata' not in coll: + coll['coll_metadata'] = {} + coll['coll_metadata']['bytes'] = 1 + # coll['coll_metadata']['total_files'] = 1 + coll['coll_metadata']['availability'] = 1 + coll['coll_metadata']['events'] = 1 + coll['coll_metadata']['is_open'] = True + coll['coll_metadata']['run_number'] = 1 + coll['coll_metadata']['did_type'] = 'DATASET' + coll['coll_metadata']['list_all_files'] = False + + if 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll['status'] = coll_status + coll['coll_type'] = CollectionType.Dataset + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_input_collections(self): + """ + *** Function called by Transformer agent. + """ + colls = [self._primary_input_collection] + self._other_input_collections + for coll_int_id in colls: + coll = self.collections[coll_int_id] + coll = self.poll_external_collection(coll) + self.collections[coll_int_id] = coll + return super(DomaLSSTWork, self).get_input_collections() + + def get_unsubmitted_inputs(self): + not_submitted_inputs = filter(lambda t: not t["submitted"], self.dependency_map) + tasks_to_check = [] + for job in not_submitted_inputs: + tasks_to_check.extend([(input["task"], input["inputname"]) for input in job["dependencies"] if not input["available"]]) + tasks_to_check_compact = {} + for task in tasks_to_check: + tasks_to_check_compact.setdefault(task[0], set()).add(task[1]) + return tasks_to_check_compact + + def set_dependency_input_available(self, taskname, inputname): + for job in self.dependency_map: + for dependency in job["dependencies"]: + if dependency["task"] == taskname and dependency["inputname"] == inputname: + dependency["available"] = True + + def update_dependencies(self): + tasks_to_check = self.get_unsubmitted_inputs() + for task, inputs in tasks_to_check.items(): + _, outputs = self.poll_panda_task(task_name=task) + for input in inputs: + if outputs.get(input, ContentStatus.Processing) == ContentStatus.Available: + self.set_dependency_input_available(task, input) + + def get_ready_inputs(self): + not_submitted_inputs = filter(lambda j: not j["submitted"], self.dependency_map) + files_to_submit = [] + for job in not_submitted_inputs: + unresolved_deps = [input for input in job["dependencies"] if not input["available"]] + if len(unresolved_deps) == 0: + files_to_submit.append(job["name"]) + return files_to_submit + + def check_dependencies(self): + self.update_dependencies() + return self.get_ready_inputs() + + def can_close(self): + not_submitted_inputs = list(filter(lambda t: not t["submitted"], self.dependency_map)) + if len(not_submitted_inputs) == 0: + return True + else: + return False + + def get_input_contents(self): + """ + Get all input contents from DDM. + """ + try: + files = self.check_dependencies() + ret_files = [] + coll = self.collections[self._primary_input_collection] + for file in files: + ret_file = {'coll_id': coll['coll_id'], + 'scope': coll['scope'], + 'name': file, # or a different file name from the dataset name + 'bytes': 1, + 'adler32': '12345678', + 'min_id': 0, + 'max_id': 1, + 'content_type': ContentType.File, + 'content_metadata': {'events': 1}} # here events is all events for eventservice, not used here. + ret_files.append(ret_file) + return ret_files + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_mapped_inputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + inputs = mapped_input_output_maps[map_id]['inputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_input = inputs[0] + for ip in inputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_input = ip + ret.append(primary_input) + return ret + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + *** Function called by Transformer agent. + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + inputs = self.get_input_contents() + mapped_inputs = self.get_mapped_inputs(mapped_input_output_maps) + mapped_inputs_scope_name = [ip['name'] for ip in mapped_inputs] + + new_inputs = [] + new_input_output_maps = {} + for ip in inputs: + ip_scope_name = ip['name'] + if ip_scope_name not in mapped_inputs_scope_name: + new_inputs.append(ip) + + # to avoid cheking new inputs if there are no new inputs anymore + if not new_inputs and self.collections[self._primary_input_collection]['status'] in [CollectionStatus.Closed]: + self.set_has_new_inputs(False) + else: + mapped_keys = mapped_input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + for ip in new_inputs: + out_ip = copy.deepcopy(ip) + out_ip['coll_id'] = self.collections[self._primary_output_collection]['coll_id'] + new_input_output_maps[next_key] = {'inputs': [ip], + 'outputs': [out_ip]} + next_key += 1 + self.logger.debug("get_new_input_output_maps, new_input_output_maps: %s" % str(new_input_output_maps)) + + for index, _inputs in new_input_output_maps.items(): + if len(_inputs['inputs']) > 0: + for item in self.dependency_map: + if item["name"] == _inputs['inputs'][0]["name"]: + item["submitted"] = True # 0 is used due to a single file pseudo input + + if self.can_close(): + self.collections[self._primary_input_collection]['coll_metadata']['is_open'] = False + self.collections[self._primary_input_collection]['status'] = CollectionStatus.Closed + + return new_input_output_maps + + def get_processing(self, input_output_maps): + """ + *** Function called by Transformer agent. + + If there is already an active processing for this work, will do nothing. + If there is no active processings, create_processing will be called. + """ + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + return None + + def create_processing(self, input_output_maps): + """ + *** Function called by Transformer agent. + + :param input_output_maps: new maps from inputs to outputs. + """ + in_files = [] + for map_id in input_output_maps: + # one map is a job which transform the inputs to outputs. + inputs = input_output_maps[map_id]['inputs'] + # outputs = input_output_maps[map_id]['outputs'] + for ip in inputs: + in_files.append(ip['name']) + + taskParamMap = {} + taskParamMap['vo'] = 'wlcg' + taskParamMap['site'] = 'BNL_OSG_1' + taskParamMap['workingGroup'] = 'lsst' + taskParamMap['nFilesPerJob'] = 1 + taskParamMap['nFiles'] = len(in_files) + taskParamMap['noInput'] = True + taskParamMap['pfnList'] = in_files + taskParamMap['taskName'] = self.task_name + taskParamMap['userName'] = 'Siarhei Padolski' + taskParamMap['taskPriority'] = 900 + taskParamMap['architecture'] = '' + taskParamMap['transUses'] = '' + taskParamMap['transHome'] = None + taskParamMap['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c' + taskParamMap['processingType'] = 'testidds' + taskParamMap['prodSourceLabel'] = 'test' + taskParamMap['taskType'] = 'test' + taskParamMap['coreCount'] = 1 + taskParamMap['skipScout'] = True + taskParamMap['cloud'] = 'US' + taskParamMap['jobParameters'] = [ + {'type': 'constant', + 'value': "echo ${IN/L}", # noqa: E501 + }, + ] + + proc = {'processing_metadata': {'internal_id': str(uuid.uuid1()), + 'task_id': None, + 'task_param': taskParamMap}} + self.add_processing_to_processings(proc) + self.active_processings.append(proc['processing_metadata']['internal_id']) + return proc + + def submit_panda_task(self, processing): + try: + from pandatools import Client + + task_param = processing['processing_metadata']['task_param'] + return_code = Client.insertTaskParams(task_param, verbose=True) + if return_code[0] == 0: + return return_code[1][1] + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc())) + return None + + def submit_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + if 'task_id' in processing['processing_metadata'] and processing['processing_metadata']['task_id']: + pass + else: + task_id = self.submit_panda_task(processing) + processing['processing_metadata']['task_id'] = task_id + + def download_payload_json(self, task_url): + response = None + try: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + req = urllib.request.Request(task_url) + response = urllib.request.urlopen(req, timeout=180, context=ctx).read() + response = json.loads(response) + except Exception or urllib.request.error as e: + raise e + return response + + def poll_panda_task(self, processing=None, task_name=None): + try: + if not self.pandamonitor: + self.pandamonitor = self.load_panda_monitor() + self.logger.info("panda server: %s" % self.pandamonitor) + + task_id = None + task_info = None + if processing: + task_id = processing['processing_metadata']['task_id'] + task_url = self.pandamonitor + '/task/?json&jeditaskid=' + str(task_id) + task_info = self.download_payload_json(task_url) + elif task_name: + task_url = self.pandamonitor + '/tasks/?taskname=' + str(task_name) + "&json" + self.logger.debug("poll_panda_task, task_url: %s" % str(task_url)) + self.logger.debug("poll_panda_task, task_url: %s" % str(self.download_payload_json(task_url))) + task_json = self.download_payload_json(task_url) + if len(task_json) > 0: + task_info = task_json[0] + else: + return "No status", {} + if not task_id: + task_id = task_info.get('jeditaskid', None) + if not task_id: + return "No status", {} + + jobs_url = self.pandamonitor + '/jobs/?json&datasets=yes&jeditaskid=' + str(task_id) + jobs_list = self.download_payload_json(jobs_url) + outputs_status = {} + for job_info in jobs_list['jobs']: + if 'jobstatus' in job_info and 'datasets' in job_info and len(job_info['datasets']) > 0: + output_index = job_info['datasets'][0]['lfn'].split(':')[1] + status = self.jobs_to_idd_ds_status(job_info['jobstatus']) + outputs_status[output_index] = status + + task_status = None + self.logger.debug("poll_panda_task, task_info: %s" % str(task_info)) + if task_info.get("task", None) is not None: + task_status = task_info["task"]["status"] + return task_status, outputs_status + except Exception as ex: + msg = "Failed to check the panda task(%s) status: %s" % (str(task_id), str(ex)) + raise exceptions.IDDSException(msg) + + def poll_processing_updates(self, processing, input_output_maps): + """ + *** Function called by Carrier agent. + """ + updated_contents = [] + update_processing = {} + self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) + + if processing: + task_status, outputs_status = self.poll_panda_task(processing=processing) + + self.logger.debug("poll_processing_updates, outputs_status: %s" % str(outputs_status)) + self.logger.debug("poll_processing_updates, task_status: %s" % str(task_status)) + + content_substatus = {'finished': 0, 'unfinished': 0} + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + key = content['name'] + if key in outputs_status: + if content.get('substatus', ContentStatus.New) != outputs_status[key]: + updated_content = {'content_id': content['content_id'], + 'substatus': outputs_status[key]} + updated_contents.append(updated_content) + content['substatus'] = outputs_status[key] + if content['substatus'] == ContentStatus.Available: + content_substatus['finished'] += 1 + else: + content_substatus['unfinished'] += 1 + + if task_status and task_status == 'done' and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Finished}} + + self.logger.debug("poll_processing_updates, update_processing: %s" % str(update_processing)) + self.logger.debug("poll_processing_updates, updated_contents: %s" % str(updated_contents)) + return update_processing, updated_contents, {} + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + for map_id in registered_input_output_maps: + outputs = registered_input_output_maps[map_id]['outputs'] + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + self.status_statistics = status_statistics + self.logger.debug("registered_input_output_maps, status_statistics: %s" % str(status_statistics)) + return status_statistics + + def syn_work_status(self, registered_input_output_maps): + self.get_status_statistics(registered_input_output_maps) + self.logger.debug("syn_work_status, self.active_processings: %s" % str(self.active_processings)) + self.logger.debug("syn_work_status, self.has_new_inputs(): %s" % str(self.has_new_inputs())) + self.logger.debug("syn_work_status, coll_metadata_is_open: %s" % str(self.collections[self.primary_input_collection]['coll_metadata']['is_open'])) + self.logger.debug("syn_work_status, primary_input_collection_status: %s" % str(self.collections[self.primary_input_collection]['status'])) + + if self.is_processings_terminated() and not self.has_new_inputs(): + keys = self.status_statistics.keys() + if ContentStatus.New.name in keys or ContentStatus.Processing.name in keys: + pass + else: + if len(keys) == 1: + if ContentStatus.Available.name in keys: + self.status = WorkStatus.Finished + else: + self.status = WorkStatus.Failed + else: + self.status = WorkStatus.SubFinished diff --git a/atlas/lib/idds/atlas/workflowv2/atlashpowork.py b/atlas/lib/idds/atlas/workflowv2/atlashpowork.py new file mode 100644 index 00000000..1180f903 --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlashpowork.py @@ -0,0 +1,751 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 + +import copy +import datetime +import json +import random +import os +import traceback + +from idds.common import exceptions +from idds.common.constants import (TransformType, CollectionType, CollectionStatus, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.common.utils import run_command +from idds.common.utils import replace_parameters_with_values +# from idds.workflowv2.work import Work +from idds.workflowv2.work import Processing +from idds.atlas.workflowv2.atlascondorwork import ATLASCondorWork +# from idds.core import (catalog as core_catalog) + + +class ATLASHPOWork(ATLASCondorWork): + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, + work_tag='hpo', exec_type='local', sandbox=None, work_id=None, + name=None, + # primary_input_collection=None, other_input_collections=None, + # output_collections=None, log_collections=None, + logger=None, + workload_id=None, + agent_attributes=None, + method=None, + container_workdir=None, + output_json=None, + opt_space=None, initial_points=None, + max_points=None, num_points_per_iteration=10): + """ + Init a work/task/transformation. + + :param setup: A string to setup the executable enviroment, it can be None. + :param executable: The executable. + :param arguments: The arguments. + :param parameters: A dict with arguments needed to be replaced. + :param work_type: The work type like data carousel, hyperparameteroptimization and so on. + :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. + :param sandbox: The sandbox. + :param work_id: The work/task id. + :param primary_input_collection: The primary input collection. + :param other_input_collections: List of the input collections. + :param output_collections: List of the output collections. + # :param workflow: The workflow the current work belongs to. + :param method: The HPO methd to use. It can be 'nevergrad', 'container' or 'sandbox'. + :param sandbox: The sandbox to be uploaded or the container path. + :param executable: The executable command. + :param arguments: The arguments for the executable. + :param container_workdir: The working directory for container. + :param opt_space: The optimization space. + :param initial_points: The initial points. + :param max_points: The maximum number of points. + :param number_points_per_iteration: The number of points to be generated per iteration. + """ + if not name: + if workload_id: + name = 'hpo.' + str(workload_id) + "." + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + else: + name = 'hpo.' + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + + primary_input_collection = {'scope': 'HPO', 'name': name} + other_input_collections = None + output_collections = [{'scope': 'HPO', 'name': name + 'output'}] + log_collections = None + + super(ATLASHPOWork, self).__init__(executable=executable, arguments=arguments, + parameters=parameters, setup=setup, work_type=TransformType.HyperParameterOpt, + exec_type=exec_type, sandbox=sandbox, work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + output_collections=output_collections, + log_collections=log_collections, + logger=logger, + agent_attributes=agent_attributes) + self.method = method + self.sandbox = sandbox + self.executable = executable + self.arguments = arguments + self.container_workdir = container_workdir + self.opt_space = opt_space + self.initial_points = initial_points + self.max_points = max_points + self.num_points_per_iteration = num_points_per_iteration + self.unfinished_points = 0 + + self.input_json = None + self.output_json = output_json + + self.finished_points = 0 + self.points_to_generate = self.num_points_per_iteration + self.point_index = 0 + self.terminated = False + self.polling_retries = 0 + + if not self.num_points_per_iteration or self.num_points_per_iteration < 0: + raise exceptions.IDDSException("num_points_per_iteration must be integer bigger than 0") + self.num_points_per_iteration = int(self.num_points_per_iteration) + + if not self.method and self.executable and 'docker' in self.executable: + self.method = 'docker' + + # if self.agent_attributes and 'atlashpowork' in self.agent_attributes: + # self.agent_attributes = self.agent_attributes['atlashpowork'] + # self.logger.info("agent_attributes: %s" % self.agent_attributes) + + # if self.agent_attributes and 'workdir' in self.agent_attributes and self.agent_attributes['workdir']: + # self.set_workdir(self.agent_attributes['workdir']) + # self.logger.info("workdir: %s" % self.get_workdir()) + + if agent_attributes: + self.set_agent_attributes(agent_attributes) + + def set_agent_attributes(self, attrs, req_attributes=None): + self.agent_attributes = attrs + + if self.agent_attributes and 'atlashpowork' in self.agent_attributes: + self.agent_attributes = self.agent_attributes['atlashpowork'] + self.logger.info("agent_attributes: %s" % self.agent_attributes) + + if self.agent_attributes and 'workdir' in self.agent_attributes and self.agent_attributes['workdir']: + if req_attributes and 'request_id' in req_attributes and 'workload_id' in req_attributes and 'transform_id' in req_attributes: + req_dir = 'request_%s_%s/transform_%s' % (req_attributes['request_id'], + req_attributes['workload_id'], + req_attributes['transform_id']) + self.set_workdir(os.path.join(self.agent_attributes['workdir'], req_dir)) + self.logger.info("workdir: %s" % self.get_workdir()) + + ####### functions for transformer ######## # noqa E266 + ###################################### # noqa E266 + + def poll_external_collection(self, coll): + try: + if coll.status in [CollectionStatus.Closed]: + if not self.terminated: + self.logger.info("Work is not terminated, reopen collection") + coll.coll_metadata['is_open'] = True + coll.status = CollectionStatus.Open + return coll + else: + coll.coll_metadata['bytes'] = 0 + coll.coll_metadata['total_files'] = 0 + coll.coll_metadata['availability'] = True + coll.coll_metadata['events'] = 0 + coll.coll_metadata['is_open'] = True + coll.coll_metadata['run_number'] = None + coll.coll_metadata['did_type'] = 'DATASET' + coll.coll_metadata['list_all_files'] = False + + if self.terminated: + self.logger.info("Work is terminated. Closing input dataset.") + coll.coll_metadata['is_open'] = False + + if self.points_to_generate <= 0: + self.logger.info("points_to_generate(%s) is equal or smaller than 0. Closing input dataset." % self.points_to_generate) + coll.coll_metadata['is_open'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + if 'did_type' in coll.coll_metadata: + if coll.coll_metadata['did_type'] == 'DATASET': + coll_type = CollectionType.Dataset + elif coll.coll_metadata['did_type'] == 'CONTAINER': + coll_type = CollectionType.Container + else: + coll_type = CollectionType.File + else: + coll_type = CollectionType.Dataset + coll.coll_metadata['coll_type'] = coll_type + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_input_collections(self): + # return [self.primary_input_collection] + self.other_input_collections + colls = [self._primary_input_collection] + self._other_input_collections + for coll_int_id in colls: + coll = self.collections[coll_int_id] + coll = self.poll_external_collection(coll) + self.collections[coll_int_id] = coll + return super(ATLASHPOWork, self).get_input_collections() + + def get_input_contents(self, point_index=1): + """ + Get all input contents from DDM. + """ + try: + if self.terminated: + return [] + + if self.unfinished_points > 0: + return [] + + ret_files = [] + coll = self.collections[self._primary_input_collection] + + if self.max_points and (self.max_points - self.finished_points < self.num_points_per_iteration): + self.points_to_generate = self.max_points - self.finished_points + + # call external processing to generate points + points = self.generate_points() + self.logger.info("points generated: %s" % str(points)) + + loss = None + for point in points: + ret_file = {'coll_id': coll.coll_id, + 'scope': coll.scope, + 'name': str(point_index), + 'bytes': 0, + 'adler32': None, + 'min_id': 0, + 'max_id': 0, + 'path': json.dumps((point, loss)), + 'content_type': ContentType.File, + 'content_metadata': {'events': 0}} + ret_files.append(ret_file) + point_index += 1 + return ret_files + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_mapped_inputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + inputs = mapped_input_output_maps[map_id]['inputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_input = inputs[0] + for ip in inputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_input = ip + ret.append(primary_input) + return ret + + def get_unfinished_points(self, mapped_input_output_maps): + counts = 0 + count_finished = 0 + for map_id in mapped_input_output_maps: + outputs = mapped_input_output_maps[map_id]['outputs'] + + for op in outputs: + if op['status'] in [ContentStatus.New]: + counts += 1 + if op['status'] in [ContentStatus.Available]: + count_finished += 1 + self.finished_points = count_finished + return counts + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + unfinished_mapped = self.get_unfinished_points(mapped_input_output_maps) + self.unfinished_points = unfinished_mapped + + mapped_inputs = self.get_mapped_inputs(mapped_input_output_maps) + mapped_inputs_scope_name = [ip['scope'] + ":" + ip['name'] for ip in mapped_inputs] + mapped_keys = mapped_input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + + inputs = self.get_input_contents(point_index=next_key) + + new_inputs = [] + new_input_output_maps = {} + for ip in inputs: + ip_scope_name = ip['scope'] + ":" + ip['name'] + if ip_scope_name not in mapped_inputs_scope_name: + new_inputs.append(ip) + + # to avoid cheking new inputs if there are no new inputs anymore + if (not new_inputs and self.collections[self._primary_input_collection] + and self.collections[self._primary_input_collection].status in [CollectionStatus.Closed]): # noqa: W503 + self.set_has_new_inputs(False) + else: + for ip in new_inputs: + out_ip = copy.deepcopy(ip) + ip['status'] = ContentStatus.Available + ip['substatus'] = ContentStatus.Available + out_ip['coll_id'] = self.collections[self._primary_output_collection].coll_id + new_input_output_maps[next_key] = {'inputs': [ip], + 'outputs': [out_ip], + 'inputs_dependency': [], + 'logs': []} + next_key += 1 + + self.unfinished_points = self.unfinished_points + len(new_inputs) + + return new_input_output_maps + + def generate_points(self): + active_processing = self.get_processing(None, without_creating=True) + if not active_processing: + if self.points_to_generate > 0: + active_processing = self.create_processing(None) + log_str = "max_points: %s, finished_points: %s, points_to_generate: %s, new processing: %s" % (self.max_points, + self.finished_points, + self.points_to_generate, + active_processing) + self.logger.info(log_str) + if active_processing: + return [] + else: + self.polling_retries += 1 + if self.polling_retries > 3: + self.terminated = True + self.set_terminated_msg("Failed to create processing") + return [] + else: + self.polling_retries += 1 + if self.polling_retries > 3: + self.terminated = True + self.set_terminated_msg("Number of points is enough(points_to_generate: %s)" % self.points_to_generate) + return [] + + if active_processing and self.is_processing_terminated(active_processing): + self.logger.info("processing terminated: %s" % active_processing) + self.reap_processing(active_processing) + # output_metadata = active_processing['output_metadata'] + output_metadata = active_processing.output_data + if output_metadata: + self.polling_retries = 0 + if self.max_points and self.max_points > self.finished_points: + self.terminated = False + return output_metadata + else: + self.polling_retries += 1 + if self.polling_retries > 3: + self.terminated = True + # processing_metadata = active_processing['processing_metadata'] + # errors = None + # if 'errors' in processing_metadata: + # errors = processing_metadata['errors'] + errors = active_processing.errors + self.set_terminated_msg("No points generated. Terminating the Work/Transformation. Detailed errors: %s" % errors) + return [] + return [] + + def get_processing(self, input_output_maps, without_creating=False): + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + # if not without_creating: + # return self.create_processing(input_output_maps) + pass + return None + + def create_processing(self, input_output_maps=[]): + processing_metadata = {'points_to_generate': self.points_to_generate} + proc = Processing(processing_metadata=processing_metadata) + self.add_processing_to_processings(proc) + self.active_processings.append(proc.internal_id) + return proc + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + for map_id in registered_input_output_maps: + outputs = registered_input_output_maps[map_id]['outputs'] + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + self.status_statistics = status_statistics + return status_statistics + + """ + def syn_collection_status(self): + input_collections = self.get_input_collections() + output_collections = self.get_output_collections() + # log_collections = self.get_log_collections() + + for input_collection in input_collections: + input_collection['total_files'] = self.finished_points + self.unfinished_points + input_collection['processed_files'] = self.finished_points + self.unfinished_points + + for output_collection in output_collections: + output_collection['total_files'] = self.finished_points + self.unfinished_points + output_collection['processed_files'] = self.finished_points + """ + + def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + super(ATLASHPOWork, self).syn_work_status(registered_input_output_maps) + self.get_status_statistics(registered_input_output_maps) + + # self.syn_collection_status() + + if self.is_processings_terminated() and not self.has_new_inputs: + if not self.is_all_outputs_flushed(registered_input_output_maps): + self.logger.warn("The processing is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform") + return + + keys = self.status_statistics.keys() + if len(keys) == 1: + if ContentStatus.Available.name in keys: + self.status = WorkStatus.Finished + else: + self.status = WorkStatus.Failed + else: + self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming + + ####### functions for carrier ######## # noqa E266 + ###################################### # noqa E266 + + def generate_processing_script_nevergrad(self, processing): + executable = self.agent_attributes['nevergrad']['executable'] + arguments = self.agent_attributes['nevergrad']['arguments'] + + param_values = {'MAX_POINTS': self.max_points, + 'NUM_POINTS': self.points_to_generate, + 'IN': self.input_json, + 'OUT': self.output_json} + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + param_values['X509_USER_PROXY_FULLNAME'] = os.environ['X509_USER_PROXY'] + param_values['X509_USER_PROXY_BASENAME'] = proxy_filename + + arguments = replace_parameters_with_values(arguments, param_values) + + script = "#!/bin/bash\n\n" + script += "executable=%s\n" % os.path.basename(executable) + script += "arguments='%s'\n" % str(arguments) + script += "input_json=%s\n" % str(self.input_json) + script += "output_json=%s\n" % str(self.output_json) + script += "\n" + + script += "env\n" + script += "echo $X509_USER_PROXY\n" + script += "\n" + + script += "echo 'user id:'\n" + script += "id\n" + script += "\n" + + script += "echo '%s' '%s'\n" % (os.path.basename(executable), str(arguments)) + script += '%s %s\n' % (os.path.basename(executable), str(arguments)) + + script += '\n' + + long_id = self.get_long_id(processing) + script_name = 'processing_%s.sh' % long_id + script_name = os.path.join(self.get_working_dir(processing), script_name) + with open(script_name, 'w') as f: + f.write(script) + run_command("chmod +x %s" % script_name) + return script_name + + def generate_processing_script_container(self, processing): + param_values = {'MAX_POINTS': self.max_points, + 'NUM_POINTS': self.points_to_generate, + 'IN': self.input_json, + 'OUT': self.output_json} + proxy_filename = 'x509up' + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + param_values['X509_USER_PROXY_FULLNAME'] = os.environ['X509_USER_PROXY'] + param_values['X509_USER_PROXY_BASENAME'] = proxy_filename + + executable = replace_parameters_with_values(self.executable, param_values) + arguments = replace_parameters_with_values(self.arguments, param_values) + + script = "#!/bin/bash\n\n" + script += "executable=%s\n" % str(executable) + script += "arguments=%s\n" % str(arguments) + script += "input_json=%s\n" % str(self.input_json) + script += "output_json=%s\n" % str(self.output_json) + script += "\n" + + script += "env\n" + script += "echo $X509_USER_PROXY\n" + script += "\n" + + script += "echo 'user id:'\n" + script += "id\n" + script += "\n" + + if self.sandbox and 'docker' in executable: + arguments = 'run --rm -v $(pwd):%s -v /cvmfs:/cvmfs -e X509_USER_PROXY=%s/%s %s ' % (self.container_workdir, self.container_workdir, proxy_filename, self.sandbox) + arguments + + script += "echo '%s' '%s'\n" % (str(executable), str(arguments)) + script += '%s %s\n' % (str(executable), str(arguments)) + + if self.sandbox and 'docker' in executable: + script += 'docker image rm -f %s\n' % self.sandbox + + script += '\n' + + long_id = self.get_long_id(processing) + script_name = 'processing_%s.sh' % long_id + script_name = os.path.join(self.get_working_dir(processing), script_name) + with open(script_name, 'w') as f: + f.write(script) + run_command("chmod +x %s" % script_name) + return script_name + + def generate_processing_script_sandbox(self, processing): + param_values = {'MAX_POINTS': self.max_points, + 'NUM_POINTS': self.points_to_generate, + 'IN': self.input_json, + 'OUT': self.output_json} + if 'X509_USER_PROXY' in os.environ and os.environ['X509_USER_PROXY']: + proxy_filename = os.path.basename(os.environ['X509_USER_PROXY']) + param_values['X509_USER_PROXY_FULLNAME'] = os.environ['X509_USER_PROXY'] + param_values['X509_USER_PROXY_BASENAME'] = proxy_filename + + executable = replace_parameters_with_values(self.executable, param_values) + arguments = replace_parameters_with_values(self.arguments, param_values) + + script = "#!/bin/bash\n\n" + script += "sandbox=%s\n" % str(self.sandbox) + script += "executable=%s\n" % str(executable) + script += "arguments=%s\n" % str(arguments) + script += "input_json=%s\n" % str(self.input_json) + script += "output_json=%s\n" % str(self.output_json) + script += "\n" + + script += "env\n" + script += "echo $X509_USER_PROXY\n" + script += "\n" + + script += "echo 'user id:'\n" + script += "id\n" + script += "\n" + + script += "wget $sandbox\n" + script += 'base_sandbox="$(basename -- $sandbox)"\n' + script += 'tar xzf $base_sandbox\n' + + script += 'chmod +x %s\n' % str(executable) + script += "echo '%s' '%s'\n" % (str(executable), str(arguments)) + script += '%s %s\n' % (str(executable), str(arguments)) + + script += '\n' + + long_id = self.get_long_id(processing) + script_name = 'processing_%s.sh' % long_id + script_name = os.path.join(self.get_working_dir(processing), script_name) + with open(script_name, 'w') as f: + f.write(script) + run_command("chmod +x %s" % script_name) + return script_name + + def generate_input_json(self, processing): + try: + from idds.core import (catalog as core_catalog) + + output_collection = self.get_output_collections()[0] + contents = core_catalog.get_contents_by_coll_id_status(coll_id=output_collection.coll_id) + points = [] + for content in contents: + # point = content['content_metadata']['point'] + point = json.loads(content['path']) + points.append(point) + + job_dir = self.get_working_dir(processing) + if 'input_json' in self.agent_attributes and self.agent_attributes['input_json']: + input_json = self.agent_attributes['input_json'] + else: + input_json = 'idds_input.json' + opt_points = {'points': points, 'opt_space': self.opt_space} + with open(os.path.join(job_dir, input_json), 'w') as f: + json.dump(opt_points, f) + return input_json + except Exception as e: + raise Exception("Failed to generate idds inputs for HPO: %s" % str(e)) + + def get_output_json(self, processing): + # job_dir = self.get_working_dir(processing) + if self.output_json: + return self.output_json + elif 'output_json' in self.agent_attributes and self.agent_attributes['output_json']: + output_json = self.agent_attributes['output_json'] + else: + output_json = 'idds_output.json' + return output_json + + def generate_processing_script(self, processing): + if not self.method: + err_msg = "Processing %s HPO method(%s) is not defined" % (processing['processing_id'], self.method) + self.logger.error(err_msg) + self.set_terminated_msg(err_msg) + self.terminated = True + return None, err_msg + + self.input_json = self.generate_input_json(processing) + self.output_json = self.get_output_json(processing) + + if self.method == 'nevergrad': + script_name = self.generate_processing_script_nevergrad(processing) + return script_name, None + elif self.method in ['container', 'docker']: + script_name = self.generate_processing_script_container(processing) + return script_name, None + elif self.method == 'sandbox': + script_name = self.generate_processing_script_sandbox(processing) + return script_name, None + else: + err_msg = "Processing %s not supported HPO method: %s" % (processing['processing_id'], self.method) + self.logger.error(err_msg) + self.set_terminated_msg(err_msg) + self.terminated = True + return None, err_msg + + def get_input_files(self, processing): + return [self.input_json] + + def get_output_files(self, processing): + return [self.output_json] + + def submit_processing(self, processing): + proc = processing['processing_metadata']['processing'] + if proc.external_id: + # if 'job_id' in processing['processing_metadata']: + pass + else: + job_id, errors = self.submit_condor_processing(processing) + # processing['processing_metadata']['job_id'] = job_id + # processing['processing_metadata']['errors'] = errors + proc.external_id = job_id + if job_id: + proc.submitted_at = datetime.datetime.utcnow() + proc.errors = errors + + def parse_processing_outputs(self, processing): + request_id = processing['request_id'] + workload_id = processing['workload_id'] + processing_id = processing['processing_id'] + + if not self.output_json: + return None, 'Request(%s)_workload(%s)_processing(%s) output_json(%s) is not defined' % (request_id, workload_id, + processing_id, self.output_json) + + job_dir = self.get_working_dir(processing) + full_output_json = os.path.join(job_dir, self.output_json) + if not os.path.exists(full_output_json): + return None, '%s is not created' % str(full_output_json) + else: + try: + with open(full_output_json, 'r') as f: + data = f.read() + outputs = json.loads(data) + if not outputs: + return outputs, "No points generated: the outputs is empty" + return outputs, None + except Exception as ex: + return None, 'Failed to load the content of %s: %s' % (str(full_output_json), str(ex)) + + def poll_processing(self, processing): + try: + proc = processing['processing_metadata']['processing'] + job_status, job_err_msg = self.poll_condor_job_status(processing, proc.external_id) + processing_outputs = None + reset_expired_at = False + if job_status in [ProcessingStatus.Finished]: + job_outputs, parser_errors = self.parse_processing_outputs(processing) + if job_outputs: + processing_status = ProcessingStatus.Finished + processing_err = None + processing_outputs = job_outputs + else: + processing_status = ProcessingStatus.Failed + processing_err = parser_errors + elif job_status in [ProcessingStatus.Failed]: + processing_status = job_status + processing_err = job_err_msg + elif self.toexpire: + processing_status = ProcessingStatus.Expired + processing_err = "The processing is expired" + elif job_status in [ProcessingStatus.Cancelled]: + processing_status = job_status + processing_err = job_err_msg + elif self.tocancel: + self.cancelled_processings.append(proc.internal_id) + processing_status = ProcessingStatus.Cancelled + processing_outputs = None + processing_err = 'Cancelled' + elif self.tosuspend: + self.suspended_processings.append(proc.internal_id) + processing_status = ProcessingStatus.Suspended + processing_outputs = None + processing_err = 'Suspend' + elif self.toresume: + # self.old_processings.append(processing['processing_metadata']['internal_id']) + # self.active_processings.clear() + # self.active_processings.remove(processing['processing_metadata']['internal_id']) + processing['processing_metadata']['resuming_at'] = datetime.datetime.utcnow() + processing_status = ProcessingStatus.Running + reset_expired_at = True + processing_outputs = None + processing_err = None + else: + processing_status = job_status + processing_err = job_err_msg + return processing_status, processing_outputs, processing_err, reset_expired_at + except Exception as ex: + self.logger.error("processing_id %s exception: %s, %s" % (processing['processing_id'], str(ex), traceback.format_exc())) + proc.retries += 1 + if proc.retries > 10: + processing_status = ProcessingStatus.Failed + else: + processing_status = ProcessingStatus.Running + return processing_status, None, None, False + + def poll_processing_updates(self, processing, input_output_maps): + processing_status, processing_outputs, processing_err, reset_expired_at = self.poll_processing(processing) + + # processing_metadata = processing['processing_metadata'] + # if not processing_metadata: + # processing_metadata = {} + # processing_metadata['errors'] = processing_err + proc = processing['processing_metadata']['processing'] + proc.errors = processing_err + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': processing_status, + 'processing_metadata': processing['processing_metadata'], + 'output_metadata': processing_outputs}} + + if reset_expired_at: + update_processing['parameters']['expired_at'] = None + processing['expired_at'] = None + updated_contents = [] + return update_processing, updated_contents, {} diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py new file mode 100644 index 00000000..a36a4f4f --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -0,0 +1,744 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + +import datetime +import os +import random +import traceback + +from rucio.client.client import Client as RucioClient +from rucio.common.exception import (CannotAuthenticate as RucioCannotAuthenticate) + +from idds.common import exceptions +from idds.common.constants import (TransformType, CollectionStatus, CollectionType, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.common.utils import extract_scope_atlas +from idds.workflowv2.work import Work, Processing +# from idds.workflowv2.workflow import Condition + + +class ATLASPandaWork(Work): + def __init__(self, task_parameters=None, + work_tag='atlas', exec_type='panda', work_id=None, + primary_input_collection=None, other_input_collections=None, + input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + logger=None, + # dependency_map=None, task_name="", + # task_queue=None, processing_type=None, + # prodSourceLabel='test', task_type='test', + # maxwalltime=90000, maxattempt=5, core_count=1, + # encode_command_line=False, + num_retries=5, + # task_log=None, + # task_cloud=None, + # task_rss=0 + ): + + super(ATLASPandaWork, self).__init__(work_type=TransformType.Processing, + work_tag=work_tag, + exec_type=exec_type, + work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + input_collections=input_collections, + output_collections=output_collections, + log_collections=log_collections, + release_inputs_after_submitting=True, + logger=logger) + self.panda_url = None + self.panda_url_ssl = None + self.panda_monitor = None + + self.task_type = 'test' + self.task_parameters = None + self.parse_task_parameters(task_parameters) + # self.logger.setLevel(logging.DEBUG) + + self.retry_number = 0 + self.num_retries = num_retries + + self.load_panda_urls() + + def my_condition(self): + if self.is_finished(): + return True + return False + + def load_panda_config(self): + panda_config = ConfigParser.SafeConfigParser() + if os.environ.get('IDDS_PANDA_CONFIG', None): + configfile = os.environ['IDDS_PANDA_CONFIG'] + if panda_config.read(configfile) == [configfile]: + return panda_config + + configfiles = ['%s/etc/panda/panda.cfg' % os.environ.get('IDDS_HOME', ''), + '/etc/panda/panda.cfg', '/opt/idds/etc/panda/panda.cfg', + '%s/etc/panda/panda.cfg' % os.environ.get('VIRTUAL_ENV', '')] + for configfile in configfiles: + if panda_config.read(configfile) == [configfile]: + return panda_config + return panda_config + + def load_panda_urls(self): + panda_config = self.load_panda_config() + # self.logger.debug("panda config: %s" % panda_config) + self.panda_url = None + self.panda_url_ssl = None + self.panda_monitor = None + + if panda_config.has_section('panda'): + if panda_config.has_option('panda', 'panda_monitor_url'): + self.panda_monitor = panda_config.get('panda', 'panda_monitor_url') + os.environ['PANDA_MONITOR_URL'] = self.panda_monitor + # self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) + if panda_config.has_option('panda', 'panda_url'): + self.panda_url = panda_config.get('panda', 'panda_url') + os.environ['PANDA_URL'] = self.panda_url + # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if panda_config.has_option('panda', 'panda_url_ssl'): + self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') + os.environ['PANDA_URL_SSL'] = self.panda_url_ssl + # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + + if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']: + self.panda_monitor = os.environ['PANDA_MONITOR_URL'] + # self.logger.debug("Panda monitor url: %s" % str(self.panda_monitor)) + if not self.panda_url and 'PANDA_URL' in os.environ and os.environ['PANDA_URL']: + self.panda_url = os.environ['PANDA_URL'] + # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: + self.panda_url_ssl = os.environ['PANDA_URL_SSL'] + # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + + def set_agent_attributes(self, attrs, req_attributes=None): + if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: + attrs['life_time'] = None + super(ATLASPandaWork, self).set_agent_attributes(attrs) + if self.agent_attributes and 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: + self.num_retries = int(self.agent_attributes['num_retries']) + + def parse_task_parameters(self, task_parameters): + if self.task_parameters: + return + elif not task_parameters: + return + self.task_parameters = task_parameters + + try: + if 'taskName' in self.task_parameters: + self.task_name = self.task_parameters['taskName'] + self.set_work_name(self.task_name) + + if 'prodSourceLabel' in self.task_parameters: + self.task_type = self.task_parameters['prodSourceLabel'] + + if 'jobParameters' in self.task_parameters: + jobParameters = self.task_parameters['jobParameters'] + for jobP in jobParameters: + if type(jobP) in [dict]: + if 'dataset' in jobP and 'param_type' in jobP: + if jobP['param_type'] == 'input': + input_c = jobP['dataset'] + scope, name = extract_scope_atlas(input_c, scopes=[]) + input_coll = {'scope': scope, 'name': name} + self.set_primary_input_collection(input_coll) + if jobP['param_type'] == 'output': + output_c = jobP['dataset'] + scope, name = extract_scope_atlas(output_c, scopes=[]) + output_coll = {'scope': scope, 'name': name} + self.add_output_collections([output_coll]) + + if 'log' in self.task_parameters: + log = self.task_parameters['log'] + dataset = log['dataset'] + scope, name = extract_scope_atlas(dataset, scopes=[]) + log_col = {'scope': scope, 'name': name} + self.add_log_collections(log_col) + + if not self.get_primary_input_collection(): + output_colls = self.get_output_collections() + output_coll = output_colls[0] + name = 'pseudo_input.' + datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) + input_coll = {'scope': output_coll.scope, 'name': name, 'type': CollectionType.PseudoDataset} + self.set_primary_input_collection(input_coll) + + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + self.add_errors(str(ex)) + + def get_rucio_client(self): + try: + client = RucioClient() + except RucioCannotAuthenticate as error: + self.logger.error(error) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc())) + return client + + def poll_external_collection(self, coll): + try: + if coll.status in [CollectionStatus.Closed]: + return coll + else: + try: + if not coll.coll_type == CollectionType.PseudoDataset: + client = self.get_rucio_client() + did_meta = client.get_metadata(scope=coll.scope, name=coll.name) + + coll.coll_metadata['bytes'] = did_meta['bytes'] + coll.coll_metadata['total_files'] = did_meta['length'] + coll.coll_metadata['availability'] = did_meta['availability'] + coll.coll_metadata['events'] = did_meta['events'] + coll.coll_metadata['is_open'] = did_meta['is_open'] + coll.coll_metadata['run_number'] = did_meta['run_number'] + coll.coll_metadata['did_type'] = did_meta['did_type'] + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + if 'did_type' in coll.coll_metadata: + if coll.coll_metadata['did_type'] == 'DATASET': + coll_type = CollectionType.Dataset + elif coll.coll_metadata['did_type'] == 'CONTAINER': + coll_type = CollectionType.Container + else: + coll_type = CollectionType.File + else: + coll_type = CollectionType.Dataset + coll.coll_metadata['coll_type'] = coll_type + coll.coll_type = coll_type + return coll + except Exception as ex: + self.logger.warn("Faield to get the dataset information(%s:%s) from Panda: %s" % (coll.scope, coll.name, str(ex))) + + coll.coll_metadata['bytes'] = 0 + coll.coll_metadata['total_files'] = 0 + coll.coll_metadata['availability'] = True + coll.coll_metadata['events'] = 0 + coll.coll_metadata['is_open'] = False + coll.coll_metadata['run_number'] = None + coll.coll_metadata['did_type'] = 'DATASET' + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + coll.coll_metadata['coll_type'] = coll.coll_type + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_input_collections(self): + """ + *** Function called by Transformer agent. + """ + colls = [self._primary_input_collection] + self._other_input_collections + for coll_int_id in colls: + coll = self.collections[coll_int_id] + # if self.is_internal_collection(coll): + # coll = self.poll_internal_collection(coll) + # else: + # coll = self.poll_external_collection(coll) + coll = self.poll_external_collection(coll) + self.collections[coll_int_id] = coll + return super(ATLASPandaWork, self).get_input_collections() + + def get_input_contents(self): + """ + Get all input contents from DDM. + """ + try: + ret_files = [] + return ret_files + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_mapped_inputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + inputs = mapped_input_output_maps[map_id]['inputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_input = inputs[0] + for ip in inputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_input = ip + ret.append(primary_input) + return ret + + def get_mapped_outputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + outputs = mapped_input_output_maps[map_id]['outputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_output = outputs[0] + for ip in outputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_output = ip + ret.append(primary_output) + return ret + + def map_file_to_content(self, coll_id, scope, name): + content = {'coll_id': coll_id, + 'scope': scope, + 'name': name, # or a different file name from the dataset name + 'bytes': 1, + 'adler32': '12345678', + 'min_id': 0, + 'max_id': 1, + 'content_type': ContentType.File, + # 'content_relation_type': content_relation_type, + # here events is all events for eventservice, not used here. + 'content_metadata': {'events': 1}} + return content + + def get_next_map_id(self, input_output_maps): + mapped_keys = input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + return next_key + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + *** Function called by Transformer agent. + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + inputs = self.get_input_contents() + mapped_inputs = self.get_mapped_inputs(mapped_input_output_maps) + mapped_inputs_scope_name = [ip['scope'] + ":" + ip['name'] for ip in mapped_inputs] + + new_inputs = [] + new_input_output_maps = {} + for ip in inputs: + ip_scope_name = ip['scope'] + ":" + ip['name'] + if ip_scope_name not in mapped_inputs_scope_name: + new_inputs.append(ip) + + # to avoid cheking new inputs if there are no new inputs anymore + if (not new_inputs and self.collections[self._primary_input_collection].status in [CollectionStatus.Closed]): # noqa: W503 + self.set_has_new_inputs(False) + else: + pass + + # self.logger.debug("get_new_input_output_maps, new_input_output_maps: %s" % str(new_input_output_maps)) + self.logger.debug("get_new_input_output_maps, new_input_output_maps len: %s" % len(new_input_output_maps)) + return new_input_output_maps + + def use_dependency_to_release_jobs(self): + """ + *** Function called by Transformer agent. + """ + return False + + def get_processing(self, input_output_maps=[], without_creating=False): + """ + *** Function called by Transformer agent. + + If there is already an active processing for this work, will do nothing. + If there is no active processings, create_processing will be called. + """ + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + if not without_creating: + # return None + return self.create_processing(input_output_maps) + return None + + def create_processing(self, input_output_maps=[]): + """ + *** Function called by Transformer agent. + + :param input_output_maps: new maps from inputs to outputs. + """ + processing_metadata = {'task_param': self.task_parameters} + proc = Processing(processing_metadata=processing_metadata) + proc.workload_id = None + self.add_processing_to_processings(proc) + self.active_processings.append(proc.internal_id) + return proc + + def submit_panda_task(self, processing): + try: + from pandatools import Client + + proc = processing['processing_metadata']['processing'] + task_param = proc.processing_metadata['task_param'] + return_code = Client.insertTaskParams(task_param, verbose=True) + if return_code[0] == 0: + try: + task_id = int(return_code[1][1]) + return task_id + except Exception as ex: + self.logger.warn("task id is not retruned: (%s) is not task id: %s" % (return_code[1][1], str(ex))) + # jediTaskID=26468582 + if return_code[1][1] and 'jediTaskID=' in return_code[1][1]: + parts = return_code[1][1].split(" ") + for part in parts: + if 'jediTaskID=' in part: + task_id = int(part.split("=")[1]) + return task_id + else: + self.logger.warn("submit_panda_task, return_code: %s" % str(return_code)) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc())) + return None + + def submit_processing(self, processing): + """ + *** Function called by Carrier agent. + """ + proc = processing['processing_metadata']['processing'] + if proc.workload_id: + # if 'task_id' in processing['processing_metadata'] and processing['processing_metadata']['task_id']: + pass + else: + task_id = self.submit_panda_task(processing) + # processing['processing_metadata']['task_id'] = task_id + # processing['processing_metadata']['workload_id'] = task_id + proc.workload_id = task_id + if task_id: + proc.submitted_at = datetime.datetime.utcnow() + + def poll_panda_task_status(self, processing): + if 'processing' in processing['processing_metadata']: + from pandatools import Client + + proc = processing['processing_metadata']['processing'] + status, task_status = Client.getTaskStatus(proc.workload_id) + if status == 0: + return task_status + else: + return 'failed' + return None + + def get_processing_status_from_panda_status(self, task_status): + if task_status in ['registered', 'defined', 'assigning']: + processing_status = ProcessingStatus.Submitting + elif task_status in ['ready', 'pending', 'scouting', 'scouted', 'prepared', 'topreprocess', 'preprocessing']: + processing_status = ProcessingStatus.Submitted + elif task_status in ['running', 'toretry', 'toincexec', 'throttled']: + processing_status = ProcessingStatus.Running + elif task_status in ['done']: + processing_status = ProcessingStatus.Finished + elif task_status in ['finished', 'paused']: + # finished, finishing, waiting it to be done + processing_status = ProcessingStatus.SubFinished + elif task_status in ['failed', 'aborted', 'broken', 'exhausted']: + # aborting, tobroken + processing_status = ProcessingStatus.Failed + else: + # finished, finishing, aborting, topreprocess, preprocessing, tobroken + # toretry, toincexec, rerefine, paused, throttled, passed + processing_status = ProcessingStatus.Submitted + return processing_status + + def get_panda_task_id(self, processing): + from pandatools import Client + + start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=10) + start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + status, results = Client.getJobIDsJediTasksInTimeRange(start_time, task_type=self.task_type, verbose=False) + if status != 0: + self.logger.warn("Error to poll latest tasks in last ten hours: %s, %s" % (status, results)) + return None + + proc = processing['processing_metadata']['processing'] + task_id = None + for req_id in results: + task_name = results[req_id]['taskName'] + if proc.workload_id is None and task_name == self.task_name: + task_id = results[req_id]['jediTaskID'] + # processing['processing_metadata']['task_id'] = task_id + # processing['processing_metadata']['workload_id'] = task_id + proc.workload_id = task_id + if task_id: + proc.submitted_at = datetime.datetime.utcnow() + + return task_id + + def poll_panda_task(self, processing=None, input_output_maps=None): + task_id = None + try: + from pandatools import Client + + if processing: + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + if task_id is None: + task_id = self.get_panda_task_id(processing) + + if task_id: + # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) + self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) + if task_info[0] != 0: + self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) + return ProcessingStatus.Submitting, [], {} + + task_info = task_info[1] + + processing_status = self.get_processing_status_from_panda_status(task_info["status"]) + + if processing_status in [ProcessingStatus.SubFinished]: + if self.retry_number < self.num_retries: + self.reactivate_processing(processing) + processing_status = ProcessingStatus.Submitted + self.retry_number += 1 + + return processing_status, [], {} + else: + return ProcessingStatus.Failed, [], {} + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + self.logger.error(msg) + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException(msg) + return ProcessingStatus.Submitting, [], {} + + def kill_processing(self, processing): + try: + if processing: + from pandatools import Client + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + # task_id = processing['processing_metadata']['task_id'] + # Client.killTask(task_id) + Client.finishTask(task_id, soft=False) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def kill_processing_force(self, processing): + try: + if processing: + from pandatools import Client + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + # task_id = processing['processing_metadata']['task_id'] + Client.killTask(task_id) + # Client.finishTask(task_id, soft=True) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def reactivate_processing(self, processing): + try: + if processing: + from pandatools import Client + # task_id = processing['processing_metadata']['task_id'] + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + + # Client.retryTask(task_id) + status, out = Client.retryTask(task_id, newParams={}) + self.logger.warn("Retry processing(%s) with task id(%s): %s, %s" % (processing['processing_id'], task_id, status, out)) + # Client.reactivateTask(task_id) + # Client.resumeTask(task_id) + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + raise exceptions.IDDSException(msg) + + def poll_processing_updates(self, processing, input_output_maps): + """ + *** Function called by Carrier agent. + """ + updated_contents = [] + update_processing = {} + reset_expired_at = False + reactive_contents = [] + # self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) + + if processing: + proc = processing['processing_metadata']['processing'] + if proc.tocancel: + self.logger.info("Cancelling processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tocancel = False + proc.polling_retries = 0 + elif proc.tosuspend: + self.logger.info("Suspending processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tosuspend = False + proc.polling_retries = 0 + elif proc.toresume: + self.logger.info("Resuming processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.reactivate_processing(processing) + reset_expired_at = True + proc.toresume = False + proc.polling_retries = 0 + proc.has_new_updates() + # reactive_contents = self.reactive_contents(input_output_maps) + # elif self.is_processing_expired(processing): + elif proc.toexpire: + self.logger.info("Expiring processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.toexpire = False + proc.polling_retries = 0 + elif proc.tofinish or proc.toforcefinish: + self.logger.info("Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.tofinish = False + proc.toforcefinish = False + proc.polling_retries = 0 + + processing_status, poll_updated_contents, new_input_output_maps = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) + self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) + self.logger.debug("poll_processing_updates, update_contents: %s" % str(poll_updated_contents)) + + if poll_updated_contents: + proc.has_new_updates() + for content in poll_updated_contents: + updated_content = {'content_id': content['content_id'], + 'substatus': content['substatus'], + 'content_metadata': content['content_metadata']} + updated_contents.append(updated_content) + + content_substatus = {'finished': 0, 'unfinished': 0} + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if content.get('substatus', ContentStatus.New) != ContentStatus.Available: + content_substatus['unfinished'] += 1 + else: + content_substatus['finished'] += 1 + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] and updated_contents: + self.logger.info("Processing %s is terminated, but there are still contents to be flushed. Waiting." % (proc.workload_id)) + # there are still polling contents, should not terminate the task. + processing_status = ProcessingStatus.Running + + if processing_status in [ProcessingStatus.SubFinished] and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + # found that a 'done' panda task has got a 'finished' status. Maybe in this case 'finished' is a transparent status. + if proc.polling_retries is None: + proc.polling_retries = 0 + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if proc.polling_retries is not None and proc.polling_retries < 3: + self.logger.info("processing %s polling_retries(%s) < 3, keep running" % (processing['processing_id'], proc.polling_retries)) + processing_status = ProcessingStatus.Running + proc.polling_retries += 1 + else: + proc.polling_retries = 0 + + if proc.in_operation_time(): + processing_status = ProcessingStatus.Running + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': processing_status}} + if reset_expired_at: + processing['expired_at'] = None + update_processing['parameters']['expired_at'] = None + proc.polling_retries = 0 + # if (processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] + # or processing['status'] in [ProcessingStatus.Resuming]): # noqa W503 + # using polling_retries to poll it again when panda may update the status in a delay(when issuing retryTask, panda will not update it without any delay). + update_processing['parameters']['status'] = ProcessingStatus.Resuming + proc.status = update_processing['parameters']['status'] + + self.logger.debug("poll_processing_updates, task: %s, update_processing: %s" % + (proc.workload_id, str(update_processing))) + self.logger.debug("poll_processing_updates, task: %s, updated_contents: %s" % + (proc.workload_id, str(updated_contents))) + self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % + (proc.workload_id, str(reactive_contents))) + return update_processing, updated_contents + reactive_contents, new_input_output_maps + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + for map_id in registered_input_output_maps: + outputs = registered_input_output_maps[map_id]['outputs'] + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + self.status_statistics = status_statistics + self.logger.debug("registered_input_output_maps, status_statistics: %s" % str(status_statistics)) + return status_statistics + + def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + super(ATLASPandaWork, self).syn_work_status(registered_input_output_maps, all_updates_flushed, output_statistics, to_release_input_contents) + # self.get_status_statistics(registered_input_output_maps) + self.status_statistics = output_statistics + + self.logger.debug("syn_work_status, self.active_processings: %s" % str(self.active_processings)) + self.logger.debug("syn_work_status, self.has_new_inputs(): %s" % str(self.has_new_inputs)) + self.logger.debug("syn_work_status, coll_metadata_is_open: %s" % + str(self.collections[self.primary_input_collection].coll_metadata['is_open'])) + self.logger.debug("syn_work_status, primary_input_collection_status: %s" % + str(self.collections[self.primary_input_collection].status)) + + self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) + self.logger.debug("syn_work_status(%s): is_input_collections_closed: %s" % (str(self.get_processing_ids()), str(self.is_input_collections_closed()))) + self.logger.debug("syn_work_status(%s): has_new_inputs: %s" % (str(self.get_processing_ids()), str(self.has_new_inputs))) + self.logger.debug("syn_work_status(%s): has_to_release_inputs: %s" % (str(self.get_processing_ids()), str(self.has_to_release_inputs()))) + self.logger.debug("syn_work_status(%s): to_release_input_contents: %s" % (str(self.get_processing_ids()), str(to_release_input_contents))) + + if self.is_processings_terminated() and self.is_input_collections_closed() and not self.has_new_inputs and not self.has_to_release_inputs() and not to_release_input_contents: + # if not self.is_all_outputs_flushed(registered_input_output_maps): + if not all_updates_flushed: + self.logger.warn("The work processings %s is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform" % str(self.get_processing_ids())) + return + + if self.is_processings_finished(): + self.status = WorkStatus.Finished + elif self.is_processings_subfinished(): + self.status = WorkStatus.SubFinished + elif self.is_processings_failed(): + self.status = WorkStatus.Failed + elif self.is_processings_expired(): + self.status = WorkStatus.Expired + elif self.is_processings_cancelled(): + self.status = WorkStatus.Cancelled + elif self.is_processings_suspended(): + self.status = WorkStatus.Suspended + elif self.is_processings_running(): + self.status = WorkStatus.Running + else: + self.status = WorkStatus.Transforming + + if self.is_processings_terminated() or self.is_processings_running() or self.is_processings_started(): + self.started = True diff --git a/atlas/lib/idds/atlas/workflowv2/atlasstageinwork.py b/atlas/lib/idds/atlas/workflowv2/atlasstageinwork.py new file mode 100644 index 00000000..89ef1b8b --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlasstageinwork.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 + +import copy +import datetime +import traceback + +from rucio.client.client import Client as RucioClient +from rucio.common.exception import (CannotAuthenticate as RucioCannotAuthenticate, + DuplicateRule as RucioDuplicateRule, + RuleNotFound as RucioRuleNotFound) + +from idds.common import exceptions +from idds.common.constants import (TransformType, CollectionType, CollectionStatus, + ContentStatus, ContentType, + ProcessingStatus, WorkStatus) +from idds.workflowv2.work import Work, Processing + + +class ATLASStageinWork(Work): + def __init__(self, executable=None, arguments=None, parameters=None, setup=None, + work_tag='stagein', exec_type='local', sandbox=None, work_id=None, + primary_input_collection=None, other_input_collections=None, input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + agent_attributes=None, + logger=None, + max_waiting_time=3600 * 7 * 24, src_rse=None, dest_rse=None, rule_id=None): + """ + Init a work/task/transformation. + + :param setup: A string to setup the executable enviroment, it can be None. + :param executable: The executable. + :param arguments: The arguments. + :param parameters: A dict with arguments needed to be replaced. + :param work_type: The work type like data carousel, hyperparameteroptimization and so on. + :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. + :param sandbox: The sandbox. + :param work_id: The work/task id. + :param primary_input_collection: The primary input collection. + :param other_input_collections: List of the input collections. + :param output_collections: List of the output collections. + # :param workflow: The workflow the current work belongs to. + :param max_waiting_time: The max waiting time to terminate the work. + :param src_rse: The source rse. + :param dest_rse: The destination rse. + :param rule_id: The rule id. + """ + super(ATLASStageinWork, self).__init__(executable=executable, arguments=arguments, + parameters=parameters, setup=setup, work_type=TransformType.StageIn, + exec_type=exec_type, sandbox=sandbox, work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + input_collections=input_collections, + output_collections=output_collections, + log_collections=log_collections, + agent_attributes=agent_attributes, + logger=logger) + self.max_waiting_time = max_waiting_time + self.src_rse = src_rse + self.dest_rse = dest_rse + self.life_time = max_waiting_time + self.rule_id = rule_id + + self.num_mapped_inputs = 0 + self.total_output_files = 0 + self.processed_output_files = 0 + self.status_statistics = {} + + def get_rucio_client(self): + try: + client = RucioClient() + except RucioCannotAuthenticate as error: + self.logger.error(error) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc())) + return client + + def poll_external_collection(self, coll): + try: + # if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: + if coll.status in [CollectionStatus.Closed]: + return coll + else: + client = self.get_rucio_client() + did_meta = client.get_metadata(scope=coll.scope, name=coll.name) + + coll.coll_metadata['bytes'] = did_meta['bytes'] + coll.coll_metadata['total_files'] = did_meta['length'] + coll.coll_metadata['availability'] = did_meta['availability'] + coll.coll_metadata['events'] = did_meta['events'] + coll.coll_metadata['is_open'] = did_meta['is_open'] + coll.coll_metadata['run_number'] = did_meta['run_number'] + coll.coll_metadata['did_type'] = did_meta['did_type'] + coll.coll_metadata['list_all_files'] = False + + if 'is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']: + coll_status = CollectionStatus.Closed + else: + coll_status = CollectionStatus.Open + coll.status = coll_status + + if 'did_type' in coll.coll_metadata: + if coll.coll_metadata['did_type'] == 'DATASET': + coll_type = CollectionType.Dataset + elif coll.coll_metadata['did_type'] == 'CONTAINER': + coll_type = CollectionType.Container + else: + coll_type = CollectionType.File + else: + coll_type = CollectionType.Dataset + coll.coll_metadata['coll_type'] = coll_type + + return coll + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return coll + + def get_input_collections(self): + # return [self.primary_input_collection] + self.other_input_collections + colls = [self._primary_input_collection] + self._other_input_collections + for coll_int_id in colls: + coll = self.collections[coll_int_id] + coll = self.poll_external_collection(coll) + self.collections[coll_int_id] = coll + return super(ATLASStageinWork, self).get_input_collections() + + def get_input_contents(self): + """ + Get all input contents from DDM. + """ + try: + ret_files = [] + rucio_client = self.get_rucio_client() + files = rucio_client.list_files(scope=self.collections[self._primary_input_collection].scope, + name=self.collections[self._primary_input_collection].name) + for file in files: + ret_file = {'coll_id': self.collections[self._primary_input_collection].coll_id, + 'scope': file['scope'], + 'name': file['name'], + 'bytes': file['bytes'], + 'adler32': file['adler32'], + 'min_id': 0, + 'max_id': file['events'], + 'content_type': ContentType.File, + 'content_metadata': {'events': file['events']}} + ret_files.append(ret_file) + return ret_files + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def get_mapped_inputs(self, mapped_input_output_maps): + ret = [] + for map_id in mapped_input_output_maps: + inputs = mapped_input_output_maps[map_id]['inputs'] + + # if 'primary' is not set, the first one is the primary input. + primary_input = inputs[0] + for ip in inputs: + if 'primary' in ip['content_metadata'] and ip['content_metadata']['primary']: + primary_input = ip + ret.append(primary_input) + return ret + + def get_new_input_output_maps(self, mapped_input_output_maps={}): + """ + New inputs which are not yet mapped to outputs. + + :param mapped_input_output_maps: Inputs that are already mapped. + """ + inputs = self.get_input_contents() + mapped_inputs = self.get_mapped_inputs(mapped_input_output_maps) + mapped_inputs_scope_name = [ip['scope'] + ":" + ip['name'] for ip in mapped_inputs] + + new_inputs = [] + new_input_output_maps = {} + for ip in inputs: + ip_scope_name = ip['scope'] + ":" + ip['name'] + if ip_scope_name not in mapped_inputs_scope_name: + new_inputs.append(ip) + + # to avoid cheking new inputs if there are no new inputs anymore + if (not new_inputs and self.collections[self._primary_input_collection].status in [CollectionStatus.Closed]): # noqa: W503 + self.set_has_new_inputs(False) + else: + mapped_keys = mapped_input_output_maps.keys() + if mapped_keys: + next_key = max(mapped_keys) + 1 + else: + next_key = 1 + for ip in new_inputs: + self.num_mapped_inputs += 1 + out_ip = copy.deepcopy(ip) + ip['status'] = ContentStatus.New + ip['substatus'] = ContentStatus.New + out_ip['coll_id'] = self.collections[self._primary_output_collection].coll_id + new_input_output_maps[next_key] = {'inputs': [ip], + 'outputs': [out_ip], + 'inputs_dependency': [], + 'logs': []} + next_key += 1 + + return new_input_output_maps + + def get_processing(self, input_output_maps, without_creating=False): + if self.active_processings: + return self.processings[self.active_processings[0]] + else: + if not without_creating: + return self.create_processing(input_output_maps) + return None + + def create_processing(self, input_output_maps=[]): + processing_metadata = {'src_rse': self.src_rse, + 'dest_rse': self.dest_rse, + 'life_time': self.life_time, + 'rule_id': self.rule_id} + proc = Processing(processing_metadata=processing_metadata) + proc.external_id = self.rule_id + if self.rule_id: + proc.submitted_at = datetime.datetime.utcnow() + + self.add_processing_to_processings(proc) + self.active_processings.append(proc.internal_id) + return proc + + def create_rule(self, processing): + try: + rucio_client = self.get_rucio_client() + ds_did = {'scope': self.collections[self._primary_input_collection].scope, + 'name': self.collections[self._primary_input_collection].name} + rule_id = rucio_client.add_replication_rule(dids=[ds_did], + copies=1, + rse_expression=self.dest_rse, + source_replica_expression=self.src_rse, + lifetime=self.lifetime, + locked=False, + grouping='DATASET', + ask_approval=False) + if type(rule_id) in (list, tuple): + rule_id = rule_id[0] + return rule_id + except RucioDuplicateRule as ex: + self.logger.warn(ex) + rules = rucio_client.list_did_rules(scope=self.collections[self._primary_input_collection].scope, + name=self.collections[self._primary_input_collection].name) + for rule in rules: + if rule['account'] == rucio_client.account and rule['rse_expression'] == self.dest_rse: + return rule['id'] + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc())) + return None + + def submit_processing(self, processing): + proc = processing['processing_metadata']['processing'] + if proc.external_id: + # if 'rule_id' in processing['processing_meta']: + pass + else: + rule_id = self.create_rule(processing) + # processing['processing_metadata']['rule_id'] = rule_id + proc.external_id = rule_id + if rule_id: + proc.submitted_at = datetime.datetime.utcnow() + + def poll_rule(self, processing): + try: + # p = processing + # rule_id = p['processing_metadata']['rule_id'] + proc = processing['processing_metadata']['processing'] + rule_id = proc.external_id + + replicases_status = {} + if rule_id: + if not isinstance(rule_id, (tuple, list)): + rule_id = [rule_id] + + rucio_client = self.get_rucio_client() + for rule_id_item in rule_id: + rule = rucio_client.get_replication_rule(rule_id=rule_id_item) + # rule['state'] + + if rule['locks_ok_cnt'] > 0: + locks = rucio_client.list_replica_locks(rule_id=rule_id_item) + for lock in locks: + scope_name = '%s:%s' % (lock['scope'], lock['name']) + if lock['state'] == 'OK': + replicases_status[scope_name] = ContentStatus.Available # 'OK' + return processing, rule['state'], replicases_status + except RucioRuleNotFound as ex: + msg = "rule(%s) not found: %s" % (str(rule_id), str(ex)) + raise exceptions.ProcessNotFound(msg) + + def poll_processing(self, processing): + try: + return self.poll_rule(processing) + except exceptions.ProcessNotFound as ex: + raise ex + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + + return processing, 'notOk', {} + + def poll_processing_updates(self, processing, input_output_maps): + try: + processing, rule_state, rep_status = self.poll_processing(processing) + + updated_contents = [] + content_substatus = {'finished': 0, 'unfinished': 0} + for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] + outputs = input_output_maps[map_id]['outputs'] + for content in inputs + outputs: + key = '%s:%s' % (content['scope'], content['name']) + if key in rep_status: + if content['substatus'] != rep_status[key]: + updated_content = {'content_id': content['content_id'], + 'substatus': rep_status[key]} + updated_contents.append(updated_content) + content['substatus'] = rep_status[key] + if content['substatus'] == ContentStatus.Available: + content_substatus['finished'] += 1 + else: + content_substatus['unfinished'] += 1 + + update_processing = {} + if rule_state == 'OK' and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Finished}} + elif self.toexpire: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Expired}} + elif self.tocancel: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Cancelled}} + elif self.tosuspend: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Suspended}} + elif self.toresume: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Running}} + update_processing['parameters']['expired_at'] = None + processing['expired_at'] = None + proc = processing['processing_metadata']['processing'] + proc.has_new_updates() + elif self.tofinish: + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.SubFinished}} + elif self.toforcefinish: + for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] + outputs = input_output_maps[map_id]['outputs'] + for content in inputs + outputs: + if content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable]: + updated_content = {'content_id': content['content_id'], + 'substatus': ContentStatus.FakeAvailable} + updated_contents.append(updated_content) + content['substatus'] = ContentStatus.FakeAvailable + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.Finished}} + + if updated_contents: + proc = processing['processing_metadata']['processing'] + proc.has_new_updates() + + return update_processing, updated_contents, {} + except exceptions.ProcessNotFound as ex: + self.logger.warn("processing_id %s not not found: %s" % (processing['processing_id'], str(ex))) + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': ProcessingStatus.SubFinished}} + return update_processing, [], {} + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise ex + + def get_status_statistics(self, registered_input_output_maps): + status_statistics = {} + + self.total_output_files = 0 + self.processed_output_file = 0 + + for map_id in registered_input_output_maps: + # inputs = registered_input_output_maps[map_id]['inputs'] + outputs = registered_input_output_maps[map_id]['outputs'] + + self.total_output_files += 1 + + for content in outputs: + if content['status'].name not in status_statistics: + status_statistics[content['status'].name] = 0 + status_statistics[content['status'].name] += 1 + + if content['status'] == ContentStatus.Available: + self.processed_output_file += 1 + + self.status_statistics = status_statistics + return status_statistics + + def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + super(ATLASStageinWork, self).syn_work_status(registered_input_output_maps) + self.get_status_statistics(registered_input_output_maps) + + # self.syn_collection_status() + + self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) + self.logger.debug("syn_work_status(%s): has_new_inputs: %s" % (str(self.get_processing_ids()), str(self.has_new_inputs))) + if self.is_processings_terminated() and not self.has_new_inputs: + if not self.is_all_outputs_flushed(registered_input_output_maps): + self.logger.warn("The processing is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform") + return + + keys = self.status_statistics.keys() + if len(keys) == 1: + if ContentStatus.Available.name in keys: + self.status = WorkStatus.Finished + else: + self.status = WorkStatus.Failed + else: + self.status = WorkStatus.SubFinished + else: + self.status = WorkStatus.Transforming + self.logger.debug("syn_work_status(%s): work.status: %s" % (str(self.get_processing_ids()), str(self.status))) From e7ca4fee0a4e6fa2900cadd51f88347e4abfca54 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 17:50:23 +0200 Subject: [PATCH 100/156] fix docs --- docs/source/general/v2/workflow.rst | 3 ++- docs/source/users/workflow_examples.rst | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/general/v2/workflow.rst b/docs/source/general/v2/workflow.rst index 6d81a46f..486682cf 100644 --- a/docs/source/general/v2/workflow.rst +++ b/docs/source/general/v2/workflow.rst @@ -24,8 +24,9 @@ Workflow .. image:: ../../images/v2/workflow_subworkflow.jpg :width: 45% :alt: Sub Workflow + .. image:: ../../images/v2/loopworkflow.jpg - :width: 45% + :width: 45% :alt: Loop Workflow A Workflow is designed to manage multiple Works(Transformations). It's a sub-class of DictClass. diff --git a/docs/source/users/workflow_examples.rst b/docs/source/users/workflow_examples.rst index 4c106074..e68b2d70 100644 --- a/docs/source/users/workflow_examples.rst +++ b/docs/source/users/workflow_examples.rst @@ -1,4 +1,4 @@ -Conditions: Examples +Workflow: Examples ============================= iDDS examples for subworkflow and loopworkflow. From f16b1eb54368e10fa00f5ae30abfb5f52ea3cb38 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 21:05:29 +0200 Subject: [PATCH 101/156] fix to propagate info from processing to workflow --- main/lib/idds/tests/core_tests.py | 12 ++++++------ workflow/lib/idds/workflowv2/work.py | 22 +++++++++++++++++++--- workflow/lib/idds/workflowv2/workflow.py | 4 ++-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index f4634428..2c4565a0 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,8 +102,8 @@ def show_works(req): print(work_ids) - -reqs = get_requests(request_id=198, with_detail=True, with_metadata=True) +""" +reqs = get_requests(request_id=203, with_detail=True, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) @@ -112,7 +112,7 @@ def show_works(req): pass sys.exit(0) - +""" """ # reqs = get_requests() @@ -128,7 +128,7 @@ def show_works(req): """ -tfs = get_transforms(request_id=194) +tfs = get_transforms(request_id=205) for tf in tfs: # print(tf) # print(tf['transform_metadata']['work'].to_dict()) @@ -136,8 +136,8 @@ def show_works(req): pass sys.exit(0) -""" +""" msgs = retrieve_messages(workload_id=25972557) number_contents = 0 for msg in msgs: @@ -156,7 +156,7 @@ def show_works(req): sys.exit(0) """ -prs = get_processings(request_id=194) +prs = get_processings(request_id=205) i = 0 for pr in prs: # if pr['request_id'] == 91: diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 494e5f56..0d1bbb7f 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -6,7 +6,7 @@ # http://www.apache.org/licenses/LICENSE-2.0OA # # Authors: -# - Wen Guan, , 2020 +# - Wen Guan, , 2020 - 2021 import copy import datetime @@ -755,7 +755,9 @@ def processings(self, value): for k in self._processings: proc = self._processings[k] if type(proc) in [Processing]: - proc_metadata[k] = {'processing_id': proc.processing_id} + proc_metadata[k] = {'processing_id': proc.processing_id, + 'workload_id': proc.workload_id, + 'external_id': proc.external_id} self.add_metadata_item('processings', proc_metadata) def refresh_work(self): @@ -772,7 +774,9 @@ def refresh_work(self): for k in self._processings: proc = self._processings[k] if type(proc) in [Processing]: - proc_metadata[k] = {'processing_id': proc.processing_id} + proc_metadata[k] = {'processing_id': proc.processing_id, + 'workload_id': proc.workload_id, + 'external_id': proc.external_id} self.add_metadata_item('processings', proc_metadata) def load_work(self): @@ -787,12 +791,24 @@ def load_work(self): if k in proc_metadata: proc_id = proc_metadata[k]['processing_id'] self._processings[k].processing_id = proc_id + if 'workload_id' in proc_metadata[k] and proc_metadata[k]['workload_id']: + self._processings[k].workload_id = proc_metadata[k]['workload_id'] + self.workload_id = proc_metadata[k]['workload_id'] + if 'external_id' in proc_metadata[k] and proc_metadata[k]['external_id']: + self._processings[k].external_id = proc_metadata[k]['external_id'] + self.external_id = proc_metadata[k]['external_id'] for k in proc_metadata: if k not in self._processings: self._processings[k] = Processing(processing_metadata={}) proc_id = proc_metadata[k]['processing_id'] self._processings[k].processing_id = proc_id self._processings[k].internal_id = k + if 'workload_id' in proc_metadata[k] and proc_metadata[k]['workload_id']: + self._processings[k].workload_id = proc_metadata[k]['workload_id'] + self.workload_id = proc_metadata[k]['workload_id'] + if 'external_id' in proc_metadata[k] and proc_metadata[k]['external_id']: + self._processings[k].external_id = proc_metadata[k]['external_id'] + self.external_id = proc_metadata[k]['external_id'] def load_metadata(self): self.load_work() diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 5f96ab74..83274f3d 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -1440,7 +1440,7 @@ def get_relation_data(self, work): next_works_data = [] for next_id in next_works: next_work = self.works[next_id] - if is_instance(next_work, Workflow): + if isinstance(next_work, Workflow): next_work_data = next_work.get_relation_map() else: next_work_data = self.get_relation_data(next_work) @@ -1707,7 +1707,7 @@ def username(self, value): @property def userdn(self): return self.template.userdn - + @userdn.setter def userdn(self, value): self.template.userdn = value From d440e36fc3d7882f80dc4f33212303ca5315bb97 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 22 Oct 2021 21:07:53 +0200 Subject: [PATCH 102/156] new version 0.8.0 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index a09efc7f..9c0d2f2e 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 072a4144..e61abec5 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.7.7 - - idds-workflow==0.7.7 \ No newline at end of file + - idds-common==0.8.0 + - idds-workflow==0.8.0 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index a09efc7f..9c0d2f2e 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index f0bb808f..967ce8d3 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.7.7 - - idds-workflow==0.7.7 \ No newline at end of file + - idds-common==0.8.0 + - idds-workflow==0.8.0 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index a09efc7f..9c0d2f2e 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 81cf0408..22603ae6 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 1c487dea..cd2d06b0 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.7.7 - - idds-workflow==0.7.7 \ No newline at end of file + - idds-common==0.8.0 + - idds-workflow==0.8.0 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index a09efc7f..9c0d2f2e 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 6a121769..cfe4c42a 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.7.7 - - idds-workflow==0.7.7 - - idds-client==0.7.7 \ No newline at end of file + - idds-common==0.8.0 + - idds-workflow==0.8.0 + - idds-client==0.8.0 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index a09efc7f..9c0d2f2e 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/website/version.py b/website/version.py index a09efc7f..9c0d2f2e 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index a09efc7f..9c0d2f2e 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.7.7" +release_version = "0.8.0" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index dfbea82f..dc6e4e8c 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.7.7 \ No newline at end of file + - idds-common==0.8.0 \ No newline at end of file From e15de85da9233167952ecacfcc04ab7a7a386f4c Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 23 Oct 2021 14:29:20 +0200 Subject: [PATCH 103/156] fix to propagate next_works --- main/lib/idds/tests/core_tests.py | 4 +--- workflow/lib/idds/workflowv2/work.py | 9 ++++++++- workflow/lib/idds/workflowv2/workflow.py | 5 +++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 2c4565a0..22c4acb0 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,8 +102,7 @@ def show_works(req): print(work_ids) -""" -reqs = get_requests(request_id=203, with_detail=True, with_metadata=True) +reqs = get_requests(request_id=211, with_detail=True, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) @@ -112,7 +111,6 @@ def show_works(req): pass sys.exit(0) -""" """ # reqs = get_requests() diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 0d1bbb7f..53747d06 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -1214,7 +1214,9 @@ def is_suspended(self): return False def add_next_work(self, work): - self.next_works.append(work) + next_works = self.next_works + next_works.append(work) + self.next_works = next_works def parse_arguments(self): try: @@ -1925,7 +1927,12 @@ def sync_work_data(self, status, substatus, work): # self.status = work.status work.work_id = self.work_id work.transforming = self.transforming + + # clerk will update next_works while transformer doesn't. + # synchronizing work metadata from transformer to clerk needs to keep it at first. + next_works = self.next_works self.metadata = work.metadata + self.next_works = next_works self.status_statistics = work.status_statistics self.processings = work.processings diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 83274f3d..9e071964 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -713,6 +713,7 @@ def works(self, value): 'external_id': work.external_id, 'status': work.status.value if work.status else work.status, 'substatus': work.substatus.value if work.substatus else work.substatus, + 'next_works': work.next_works, 'transforming': work.transforming} self.add_metadata_item('works', work_metadata) @@ -732,6 +733,7 @@ def refresh_works(self): 'external_id': work.external_id, 'status': work.status.value if work.status else work.status, 'substatus': work.substatus.value if work.substatus else work.substatus, + 'next_works': work.next_works, 'transforming': work.transforming} if work.last_updated_at and (not self.last_updated_at or work.last_updated_at > self.last_updated_at): self.last_updated_at = work.last_updated_at @@ -748,6 +750,7 @@ def load_works(self): self._works[k].transforming = work_metadata[k]['transforming'] self._works[k].status = WorkStatus(work_metadata[k]['status']) if work_metadata[k]['status'] else work_metadata[k]['status'] self._works[k].substatus = WorkStatus(work_metadata[k]['substatus']) if work_metadata[k]['substatus'] else work_metadata[k]['substatus'] + self._works[k].next_works = work_metadata[k]['next_works'] if 'next_works' in work_metadata[k] else [] elif work_metadata[k]['type'] == 'workflow': self._works[k].metadata = work_metadata[k]['metadata'] @@ -1413,6 +1416,8 @@ def sync_works(self): log_str += ", num_suspended_works: %s" % self.num_suspended_works self.log_debug(log_str) + self.refresh_works() + def resume_works(self): self.num_subfinished_works = 0 self.num_finished_works = 0 From c86fe33b798ae7c179ab6f66b4c98b332cbda864 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 23 Oct 2021 15:04:08 +0200 Subject: [PATCH 104/156] add docs for monitor api --- docs/source/index.rst | 1 + docs/source/users/monitorapi_examples.rst | 392 ++++++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 docs/source/users/monitorapi_examples.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index bd5a9a49..5b603ccf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,6 +57,7 @@ User Documentation users/cli_examples users/condition_examples users/workflow_examples + users/monitorapi_examples users/admin_guides users/contributing diff --git a/docs/source/users/monitorapi_examples.rst b/docs/source/users/monitorapi_examples.rst new file mode 100644 index 00000000..e0d6ad49 --- /dev/null +++ b/docs/source/users/monitorapi_examples.rst @@ -0,0 +1,392 @@ +Monitor API: Examples +============================= + +iDDS provides a group of monitor APIs which returns JSON outputs. + +Request information +~~~~~~~~~~~~~~~~~~~~~~~~ + +It returns a summary information monthly(accumulated and not accumulated). + +.. code-block:: python + + curl https://hostname:443/idds/monitor_request// + curl https://hostname:443/idds/monitor_request/null/null + + { + "month_acc_status": { + "Finished": { + "2021-05": 1, + "2021-06": 1, + "2021-07": 1, + "2021-08": 13, + "2021-09": 13, + "2021-10": 28 + }, + "Total": { + "2021-05": 8, + "2021-06": 8, + "2021-07": 8, + "2021-08": 52, + "2021-09": 54, + "2021-10": 80 + }, + }, + "month_status": { + "Finished": { + "2021-05": 1, + "2021-06": 0, + "2021-07": 0, + "2021-08": 12, + "2021-09": 0, + "2021-10": 15 + }, + "SubFinished": { + "2021-05": 0, + "2021-06": 0, + "2021-07": 0, + "2021-08": 15, + "2021-09": 2, + "2021-10": 1 + }, + "Total": { + "2021-05": 8, + "2021-06": 0, + "2021-07": 0, + "2021-08": 44, + "2021-09": 2, + "2021-10": 26 + }, + }, + "total": 80 + } + + +Here it returns the detail of requests. + +.. code-block:: python + + curl https://hostname:443/idds/monitor///true/false/false + curl https://hostname:443/idds/monitor/null/null/true/false/false + curl https://hostname:443/idds/monitor/210/null/true/false/false + [ + { + "created_at": "Fri, 22 Oct 2021 22:17:42 UTC", + "input_coll_bytes": 3, + "input_processed_files": 14, + "input_processing_files": 0, + "input_total_files": 14, + "output_coll_bytes": 14, + "output_processed_files": 14, + "output_processing_files": 0, + "output_total_files": 14, + "request_id": 210, + "status": "Finished", + "transforms": { + "Finished": 3 + }, + "updated_at": "Fri, 22 Oct 2021 23:55:13 UTC", + "workload_id": 1634941059 + } + ] + +Transform summary +~~~~~~~~~~~~~~~~~~~~~~~ + +It returns a summary information monthly(accumulated and not accumulated). + +.. code-block:: python + + curl https://hostname:443/idds/monitor_transform// + curl https://hostname:443/idds/monitor_transform/null/null + curl https://hostname:443/idds/monitor_transform/210/null + { + "month_acc_processed_bytes": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + }, + "month_acc_processed_bytes_by_type": { + "Processing": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + } + }, + "month_acc_processed_files": { + "Finished": { + "2021-10": 14 + }, + "Total": { + "2021-10": 14 + } + }, + "month_acc_processed_files_by_type": { + "Processing": { + "Finished": { + "2021-10": 14 + }, + "Total": { + "2021-10": 14 + } + } + }, + "month_acc_status": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + }, + "month_acc_status_dict_by_type": { + "Processing": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + } + }, + "month_processed_bytes": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + }, + "month_processed_bytes_by_type": { + "Processing": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + } + }, + "month_processed_files": { + "Finished": { + "2021-10": 14 + }, + "Total": { + "2021-10": 14 + } + }, + "month_processed_files_by_type": { + "Processing": { + "Finished": { + "2021-10": 14 + }, + "Total": { + "2021-10": 14 + } + } + }, + "month_status": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + }, + "month_status_dict_by_type": { + "Processing": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + } + }, + "total": 3, + "total_bytes": 17, + "total_files": 14 + } + +Here it returns the list of detailed transforms. + +.. code-block:: python + + curl https://hostname:443/idds/monitor///false/true/false + curl https://hostname:443/idds/monitor/null/null/false/true/false + curl https://hostname:443/idds/monitor/210/null/false/true/false + [ + { + "errors": { + "msg": "" + }, + "input_coll_bytes": 1, + "input_processed_files": 3, + "input_processing_files": 0, + "input_total_files": 3, + "output_coll_bytes": 3, + "output_coll_name": "pseudo_output_collection#2", + "output_coll_scope": "pseudo_dataset", + "output_processed_files": 3, + "output_processing_files": 0, + "output_total_files": 3, + "request_id": 210, + "transform_created_at": "Fri, 22 Oct 2021 22:50:16 UTC", + "transform_finished_at": "Fri, 22 Oct 2021 23:20:43 UTC", + "transform_id": 2445, + "transform_status": "Finished", + "transform_type": "Processing", + "transform_updated_at": "Fri, 22 Oct 2021 23:20:43 UTC", + "transform_workload_id": 7169, + "workload_id": 1634941059 + }, + { + "errors": { + "msg": "" + }, + "input_coll_bytes": 1, + "input_processed_files": 6, + "input_processing_files": 0, + "input_total_files": 6, + "output_coll_bytes": 6, + "output_coll_name": "pseudo_output_collection#1", + "output_coll_scope": "pseudo_dataset", + "output_processed_files": 6, + "output_processing_files": 0, + "output_total_files": 6, + "request_id": 210, + "transform_created_at": "Fri, 22 Oct 2021 22:17:45 UTC", + "transform_finished_at": "Fri, 22 Oct 2021 22:46:15 UTC", + "transform_id": 2444, + "transform_status": "Finished", + "transform_type": "Processing", + "transform_updated_at": "Fri, 22 Oct 2021 22:46:15 UTC", + "transform_workload_id": 7168, + "workload_id": 1634941059 + }, + { + "errors": { + "msg": "" + }, + "input_coll_bytes": 1, + "input_processed_files": 5, + "input_processing_files": 0, + "input_total_files": 5, + "output_coll_bytes": 5, + "output_coll_name": "pseudo_output_collection#3", + "output_coll_scope": "pseudo_dataset", + "output_processed_files": 5, + "output_processing_files": 0, + "output_total_files": 5, + "request_id": 210, + "transform_created_at": "Fri, 22 Oct 2021 23:24:46 UTC", + "transform_finished_at": "Fri, 22 Oct 2021 23:53:15 UTC", + "transform_id": 2446, + "transform_status": "Finished", + "transform_type": "Processing", + "transform_updated_at": "Fri, 22 Oct 2021 23:53:15 UTC", + "transform_workload_id": 7170, + "workload_id": 1634941059 + } + ] + +Processing information +~~~~~~~~~~~~~~~~~~~~~~ + +Here it returns a summary information monthly(accumulated and not accumulated). + +.. code-block:: python + + curl https://hostname:443/idds/monitor_processing// + curl https://hostname:443/idds/monitor_processing/null/null + curl https://hostname:443/idds/monitor_processing/210/null + { + "month_acc_status": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + }, + "month_status": { + "Finished": { + "2021-10": 3 + }, + "Total": { + "2021-10": 3 + } + }, + "total": 3 + } + +.. code-block:: python + + curl https://hostname:443/idds/monitor///false/false/true + curl https://hostname:443/idds/monitor/null/null/false/false/true + curl https://hostname:443/idds/monitor/210/null/false/false/true + [ + { + "processing_created_at": "Fri, 22 Oct 2021 23:24:50 UTC", + "processing_finished_at": "Fri, 22 Oct 2021 23:53:10 UTC", + "processing_id": 1230, + "processing_status": "Finished", + "processing_updated_at": "Fri, 22 Oct 2021 23:53:10 UTC", + "request_id": 210, + "workload_id": 7170 + }, + { + "processing_created_at": "Fri, 22 Oct 2021 22:50:20 UTC", + "processing_finished_at": "Fri, 22 Oct 2021 23:18:40 UTC", + "processing_id": 1229, + "processing_status": "Finished", + "processing_updated_at": "Fri, 22 Oct 2021 23:18:40 UTC", + "request_id": 210, + "workload_id": 7169 + }, + { + "processing_created_at": "Fri, 22 Oct 2021 22:17:49 UTC", + "processing_finished_at": "Fri, 22 Oct 2021 22:46:05 UTC", + "processing_id": 1228, + "processing_status": "Finished", + "processing_updated_at": "Fri, 22 Oct 2021 22:46:05 UTC", + "request_id": 210, + "workload_id": 7168 + } + ] + +DAG relationships +~~~~~~~~~~~~~~~~~~~~~ + +Here it returns the request information with dag relationships. +It returns a list of works. For every work, it returns "work" for work data and "next_works" for its followings(if existing). + +.. code-block:: python + + curl https://hostname:443/idds/monitor_request_relation// + curl https://hostname:443/idds/monitor_request_relation/212/null + + [{ + "relation_map": [ + { + "next_works": [ + { + "work": { + "external_id": null, + "workload_id": 7175 + } + } + ], + "work": { + "external_id": null, + "workload_id": 7174 + } + } + ], + }] From 4b1bd3a065e7d3809cb223c276a44b02fd044f7d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 23 Oct 2021 15:04:58 +0200 Subject: [PATCH 105/156] add test domapanda workflow --- .../lib/idds/tests/test_domapanda_workflow.py | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 main/lib/idds/tests/test_domapanda_workflow.py diff --git a/main/lib/idds/tests/test_domapanda_workflow.py b/main/lib/idds/tests/test_domapanda_workflow.py new file mode 100644 index 00000000..70ec5a73 --- /dev/null +++ b/main/lib/idds/tests/test_domapanda_workflow.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Sergey Padolski, , 2021 +# - Wen Guan, , 2021 + + +""" +Test client. +""" + +import string +import random + +# import traceback + +# from rucio.client.client import Client as Rucio_Client +# from rucio.common.exception import CannotAuthenticate + +# from idds.client.client import Client +from idds.client.clientmanager import ClientManager +# from idds.common.constants import RequestType, RequestStatus +from idds.common.utils import get_rest_host +# from idds.tests.common import get_example_real_tape_stagein_request +# from idds.tests.common import get_example_prodsys2_tape_stagein_request + +# from idds.workflowv2.work import Work, Parameter, WorkStatus +# from idds.workflowv2.workflow import Condition, Workflow +from idds.workflowv2.workflow import Workflow +# from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork +from idds.doma.workflowv2.domapandawork import DomaPanDAWork + + +task_queue = 'DOMA_LSST_GOOGLE_TEST' + + +def randStr(chars=string.ascii_lowercase + string.digits, N=10): + return ''.join(random.choice(chars) for _ in range(N)) + + +class PanDATask(object): + name = None + step = None + dependencies = [] + + +def setup_workflow(): + + taskN1 = PanDATask() + taskN1.step = "step1" + taskN1.name = taskN1.step + "_" + randStr() + taskN1.dependencies = [ + {"name": "00000" + str(k), + "dependencies": [], + "submitted": False} for k in range(6) + ] + + taskN2 = PanDATask() + taskN2.step = "step2" + taskN2.name = taskN2.step + "_" + randStr() + taskN2.dependencies = [ + { + "name": "000010", + "dependencies": [{"task": taskN1.name, "inputname": "000001", "available": False}, + {"task": taskN1.name, "inputname": "000002", "available": False}], + "submitted": False + }, + { + "name": "000011", + "dependencies": [{"task": taskN1.name, "inputname": "000001", "available": False}, + {"task": taskN1.name, "inputname": "000002", "available": False}], + "submitted": False + }, + { + "name": "000012", + "dependencies": [{"task": taskN1.name, "inputname": "000001", "available": False}, + {"task": taskN1.name, "inputname": "000002", "available": False}], + "submitted": False + } + ] + + taskN3 = PanDATask() + taskN3.step = "step3" + taskN3.name = taskN3.step + "_" + randStr() + taskN3.dependencies = [ + { + "name": "000020", + "dependencies": [], + "submitted": False + }, + { + "name": "000021", + "dependencies": [{"task": taskN2.name, "inputname": "000010", "available": False}, + {"task": taskN2.name, "inputname": "000011", "available": False}], + "submitted": False + }, + { + "name": "000022", + "dependencies": [{"task": taskN2.name, "inputname": "000011", "available": False}, + {"task": taskN2.name, "inputname": "000012", "available": False}], + "submitted": False + }, + { + "name": "000023", + "dependencies": [], + "submitted": False + }, + { + "name": "000024", + "dependencies": [{"task": taskN3.name, "inputname": "000021", "available": False}, + {"task": taskN3.name, "inputname": "000023", "available": False}], + "submitted": False + }, + ] + + work1 = DomaPanDAWork(executable='echo', + primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#1'}, + output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#1'}], + log_collections=[], dependency_map=taskN1.dependencies, + task_name=taskN1.name, task_queue=task_queue, + encode_command_line=True, + task_log={"dataset": "PandaJob_#{pandaid}/", + "destination": "local", + "param_type": "log", + "token": "local", + "type": "template", + "value": "log.tgz"}, + task_cloud='LSST') + work2 = DomaPanDAWork(executable='echo', + primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#2'}, + output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#2'}], + log_collections=[], dependency_map=taskN2.dependencies, + task_name=taskN2.name, task_queue=task_queue, + task_log={"dataset": "PandaJob_#{pandaid}/", + "destination": "local", + "param_type": "log", + "token": "local", + "type": "template", + "value": "log.tgz"}, + task_cloud='LSST') + work3 = DomaPanDAWork(executable='echo', + primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#3'}, + output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#3'}], + log_collections=[], dependency_map=taskN3.dependencies, + task_name=taskN3.name, task_queue=task_queue, + task_log={"dataset": "PandaJob_#{pandaid}/", + "destination": "local", + "param_type": "log", + "token": "local", + "type": "template", + "value": "log.tgz"}, + task_cloud='LSST') + + pending_time = 0.5 + # pending_time = None + workflow = Workflow(pending_time=pending_time) + workflow.add_work(work1) + workflow.add_work(work2) + workflow.add_work(work3) + return workflow + + +if __name__ == '__main__': + host = get_rest_host() + workflow = setup_workflow() + + wm = ClientManager(host=host) + request_id = wm.submit(workflow) + print(request_id) From 476a0263c242bb073c9fbd290f87657649b44111 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 23 Oct 2021 15:05:16 +0200 Subject: [PATCH 106/156] new version 0.8.1 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index e61abec5..600bee82 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.0 - - idds-workflow==0.8.0 \ No newline at end of file + - idds-common==0.8.1 + - idds-workflow==0.8.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 967ce8d3..35ef4733 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.0 - - idds-workflow==0.8.0 \ No newline at end of file + - idds-common==0.8.1 + - idds-workflow==0.8.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 22603ae6..2c713f6b 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index cd2d06b0..d63e554c 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.0 - - idds-workflow==0.8.0 \ No newline at end of file + - idds-common==0.8.1 + - idds-workflow==0.8.1 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index cfe4c42a..8df11fe8 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.0 - - idds-workflow==0.8.0 - - idds-client==0.8.0 \ No newline at end of file + - idds-common==0.8.1 + - idds-workflow==0.8.1 + - idds-client==0.8.1 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/website/version.py b/website/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 9c0d2f2e..3cc44fd3 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.0" +release_version = "0.8.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index dc6e4e8c..6f211a84 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.0 \ No newline at end of file + - idds-common==0.8.1 \ No newline at end of file From f46bac4b4c2c17ae0a04c386c9977ddab44ad142 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 23 Oct 2021 18:48:09 +0200 Subject: [PATCH 107/156] add docs for monitor api --- docs/source/users/monitorapi_examples.rst | 33 ++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/source/users/monitorapi_examples.rst b/docs/source/users/monitorapi_examples.rst index e0d6ad49..e59d7c73 100644 --- a/docs/source/users/monitorapi_examples.rst +++ b/docs/source/users/monitorapi_examples.rst @@ -90,7 +90,7 @@ Here it returns the detail of requests. } ] -Transform summary +Transform information ~~~~~~~~~~~~~~~~~~~~~~~ It returns a summary information monthly(accumulated and not accumulated). @@ -373,6 +373,7 @@ It returns a list of works. For every work, it returns "work" for work data and curl https://hostname:443/idds/monitor_request_relation/212/null [{ + ...... "relation_map": [ { "next_works": [ @@ -389,4 +390,34 @@ It returns a list of works. For every work, it returns "work" for work data and } } ], + ...... + }] + +If there is a loop workflow or a sub loop wookflow. The returned format will be: + +.. code-block:: python + [{ + ...... + "relation_map": [ + { + "next_works": [ + { + "work": { + "external_id": null, + "workload_id": 7175 + }, + "next_works": [ + {"1": [{"work": <>, "next_works": <>}, ...], # the first loop for a loop workflow. + "2": <> # the second loop for a loop workflow. + } + ] + } + ], + "work": { + "external_id": null, + "workload_id": 7174 + } + } + ], + ...... }] From 774e7b76c1662aeac3c357fe6804d24bbc9b05f6 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 23 Oct 2021 18:48:45 +0200 Subject: [PATCH 108/156] new version 0.8.2 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 3cc44fd3..3085dbb1 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 600bee82..ce840291 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.1 - - idds-workflow==0.8.1 \ No newline at end of file + - idds-common==0.8.2 + - idds-workflow==0.8.2 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 3cc44fd3..3085dbb1 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 35ef4733..36c4a2c5 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.1 - - idds-workflow==0.8.1 \ No newline at end of file + - idds-common==0.8.2 + - idds-workflow==0.8.2 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 3cc44fd3..3085dbb1 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 2c713f6b..11f5e173 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index d63e554c..d2bd7066 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.1 - - idds-workflow==0.8.1 \ No newline at end of file + - idds-common==0.8.2 + - idds-workflow==0.8.2 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 3cc44fd3..3085dbb1 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 8df11fe8..8640646e 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.1 - - idds-workflow==0.8.1 - - idds-client==0.8.1 \ No newline at end of file + - idds-common==0.8.2 + - idds-workflow==0.8.2 + - idds-client==0.8.2 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 3cc44fd3..3085dbb1 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/website/version.py b/website/version.py index 3cc44fd3..3085dbb1 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 3cc44fd3..3085dbb1 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.1" +release_version = "0.8.2" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 6f211a84..683c251d 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.1 \ No newline at end of file + - idds-common==0.8.2 \ No newline at end of file From 413518e49d04576e2a11274f4cb83675be0183af Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 26 Oct 2021 17:28:44 +0200 Subject: [PATCH 109/156] add test doma workflow --- main/lib/idds/tests/test_domapanda_workflow.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main/lib/idds/tests/test_domapanda_workflow.py b/main/lib/idds/tests/test_domapanda_workflow.py index 70ec5a73..9c6d75de 100644 --- a/main/lib/idds/tests/test_domapanda_workflow.py +++ b/main/lib/idds/tests/test_domapanda_workflow.py @@ -31,7 +31,7 @@ # from idds.workflowv2.work import Work, Parameter, WorkStatus # from idds.workflowv2.workflow import Condition, Workflow -from idds.workflowv2.workflow import Workflow +from idds.workflowv2.workflow import Workflow, Condition # from idds.atlas.workflowv2.atlasstageinwork import ATLASStageinWork from idds.doma.workflowv2.domapandawork import DomaPanDAWork @@ -156,12 +156,17 @@ def setup_workflow(): "value": "log.tgz"}, task_cloud='LSST') + cond1 = Condition(cond=work1.is_finished, true_work=work2) + cond2 = Condition(cond=work2.is_finished, true_work=work3) + pending_time = 0.5 # pending_time = None workflow = Workflow(pending_time=pending_time) workflow.add_work(work1) workflow.add_work(work2) workflow.add_work(work3) + workflow.add_condition(cond1) + workflow.add_condition(cond2) return workflow From ed3e8198991eaca1aef64e4c3150f60c81e58af2 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 14:15:42 +0100 Subject: [PATCH 110/156] fix monitor setup --- monitor/setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/monitor/setup.py b/monitor/setup.py index daa56190..3c3c4517 100644 --- a/monitor/setup.py +++ b/monitor/setup.py @@ -105,7 +105,10 @@ def get_data_files(dest, src): data.append((dest, get_files(src))) for root, dirs, files in os.walk(src): if 'dist' in root or 'build' in root or 'egg-info' in root: - continue + # continue + pass + if root.endswith('monitor/dist') or root.endswith('monitor/build') or 'egg-info' in root: + # continue pass for idir in dirs: if idir == 'dist' or idir == 'build' or idir.endswith('.egg-info'): From fe456f83a9fabfbe4d1db60ed977b21edc8904c3 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 14:16:21 +0100 Subject: [PATCH 111/156] to support global parameters --- workflow/lib/idds/workflowv2/work.py | 5 ++++ workflow/lib/idds/workflowv2/workflow.py | 30 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 53747d06..9aa84cbe 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -1002,6 +1002,11 @@ def get_work_name(self): def get_is_template(self): self.is_template + def sync_global_parameters(self, global_parameters): + if global_parameters: + for key in global_parameters: + setattr(self, key, global_parameters[key]) + def setup_logger(self): """ Setup logger diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 9e071964..4cc89c95 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -549,6 +549,8 @@ def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None self.parameter_links_source = {} self.parameter_links_destination = {} + self.global_parameters = {} + super(WorkflowBase, self).__init__() self.internal_id = str(uuid.uuid4())[:8] @@ -790,6 +792,32 @@ def load_work_conditions(self): # work_conds = self.get_metadata_item('work_conds', {}) # self._work_conds = work_conds + @property + def global_parameters(self): + self._global_parameters = self.get_metadata_item('gp', {}) + return self._global_parameters + + @global_parameters.setter + def global_parameters(self, value): + self._global_parameters = value + gp_metadata = {} + if self._global_parameters: + for key in self._global_parameters: + if key.startswith("user_"): + gp_metadata[key] = self._global_parameters[key] + else: + self.logger.warn("Only parameters start with 'user_' can be set as global parameters. The parameter '%s' will be ignored." % (key)) + self.add_metadata_item('gp', gp_metadata) + + def set_global_parameters(self, value): + self.global_parameters = value + + def sync_global_parameters_from_work(self, work): + if self.global_parameters: + for key in self.global_parameters: + if hasattr(work, key): + self.global_parameters[key] = getattr(work, key) + @property def loop_condition(self): return self._loop_condition @@ -1005,6 +1033,7 @@ def get_new_work_to_run(self, work_id, new_parameters=None): work.sequence_id = self.num_total_works work.initialize_work() + work.sync_global_parameters(self.global_parameters) work.num_run = self.num_run works = self.works self.works = works @@ -1364,6 +1393,7 @@ def sync_works(self): if work.is_terminated(): self.set_source_parameters(work.get_internal_id()) + self.sync_global_parameters_from_work(work) if work.get_internal_id() in self.work_conds: self.log_debug("Work %s has condition dependencies %s" % (work.get_internal_id(), From 59cda5f6240a63d8e712671ae99ac6ab175fd040 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:19:23 +0100 Subject: [PATCH 112/156] to support Condition in _conditions --- workflow/lib/idds/workflowv2/work.py | 10 ++++-- workflow/lib/idds/workflowv2/workflow.py | 45 ++++++++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 9aa84cbe..b97d2568 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -951,7 +951,10 @@ def primary_output_collection(self, value): @property def input_collections(self): - keys = [self._primary_input_collection] + self._other_input_collections + if self._primary_input_collection: + keys = [self._primary_input_collection] + self._other_input_collections + else: + keys = self._other_input_collections return [self.collections[k] for k in keys] @input_collections.setter @@ -973,7 +976,10 @@ def input_collections(self, value): @property def output_collections(self): - keys = [self._primary_output_collection] + self._other_output_collections + if self._primary_output_collection: + keys = [self._primary_output_collection] + self._other_output_collections + else: + keys = self._other_output_collections return [self.collections[k] for k in keys] @output_collections.setter diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 4cc89c95..cb3d318c 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -199,10 +199,24 @@ def to_dict(self): new_value = [] for cond in value: if inspect.ismethod(cond): - new_cond = {'idds_method': cond.__name__, - 'idds_method_internal_id': cond.__self__.get_internal_id()} + if isinstance(cond.__self__, Work): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id()} + elif isinstance(cond.__self__, CompositeCondition): + new_cond = {'idds_method': cond.__name__, + 'idds_method_condition': cond.__self__.to_dict()} + elif isinstance(cond.__self__, Workflow): + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id()} + else: + new_cond = {'idds_method': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id()} else: - new_cond = cond + if hasattr(cond, '__self__'): + new_cond = {'idds_attribute': cond.__name__, + 'idds_method_internal_id': cond.__self__.get_internal_id()} + else: + new_cond = cond new_value.append(new_cond) value = new_value elif key in ['_true_works', '_false_works']: @@ -240,6 +254,17 @@ def load_conditions(self, works): else: self.logger.error("Work cannot be found for %s" % (internal_id)) new_cond = cond + elif 'idds_attribute' in cond and 'idds_method_internal_id' in cond: + internal_id = cond['idds_method_internal_id'] + work = self.get_work_from_id(internal_id, works) + if work is not None: + new_cond = getattr(work, cond['idds_attribute']) + else: + self.logger.error("Work cannot be found for %s" % (internal_id)) + new_cond = cond + elif 'idds_method' in cond and 'idds_method_condition' in cond: + new_cond = cond['idds_method_condition'] + new_cond = getattr(new_cond, cond['idds_method']) else: new_cond = cond new_conditions.append(new_cond) @@ -289,7 +314,10 @@ def all_condition_ids(self): works = [] for cond in self.conditions: if inspect.ismethod(cond): - works.append(cond.__self__.get_internal_id()) + if isinstance(cond.__self__, Work) or isinstance(cond.__self__, Workflow): + works.append(cond.__self__.get_internal_id()) + elif isinstance(cond.__self__, CompositeCondition): + works = works + cond.__self__.all_condition_ids() else: self.logger.error("cond cannot be recognized: %s" % str(cond)) works.append(cond) @@ -302,7 +330,10 @@ def all_pre_works(self): works = [] for cond in self.conditions: if inspect.ismethod(cond): - works.append(cond.__self__) + if isinstance(cond.__self__, Work) or isinstance(cond.__self__, Workflow): + works.append(cond.__self__) + elif isinstance(cond.__self__, CompositeCondition): + works = works + cond.__self__.all_pre_works() else: self.logger.error("cond cannot be recognized: %s" % str(cond)) works.append(cond) @@ -549,7 +580,7 @@ def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None self.parameter_links_source = {} self.parameter_links_destination = {} - self.global_parameters = {} + self._global_parameters = {} super(WorkflowBase, self).__init__() @@ -613,6 +644,8 @@ def __init__(self, name=None, workload_id=None, lifetime=None, pending_time=None self.num_run = None + self.global_parameters = {} + """ self._running_data_names = [] for name in ['internal_id', 'template_work_id', 'workload_id', 'work_sequence', 'terminated_works', From ebbd24e7d19875bf107981b8ce62fea375f4d546 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:19:52 +0100 Subject: [PATCH 113/156] to test Condition in _conditions --- .../idds/tests/test_workflow_condition_v2.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/main/lib/idds/tests/test_workflow_condition_v2.py b/main/lib/idds/tests/test_workflow_condition_v2.py index 3dc11784..a6007b84 100644 --- a/main/lib/idds/tests/test_workflow_condition_v2.py +++ b/main/lib/idds/tests/test_workflow_condition_v2.py @@ -405,6 +405,82 @@ def test_condition(self): work5.status = WorkStatus.New work1.status = WorkStatus.New + # multiple conditions + # cond8 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond8 = Condition(cond=work1.is_finished) + cond9 = CompositeCondition(conditions=[work4.is_finished, cond8.is_condition_true], true_works=[work6], false_works=work7) + + works = cond9.all_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4, work6, work7]) + works = cond9.all_pre_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4]) + works = cond9.all_next_works() + works.sort(key=lambda x: x.work_id) + # print([w.work_id for w in works]) + assert(works == [work6, work7]) + cond_status = cond9.get_condition_status() + assert(cond_status is False) + + work4.status = WorkStatus.Finished + cond_status = cond9.get_condition_status() + assert(cond_status is False) + work1.status = WorkStatus.Finished + cond_status = cond9.get_condition_status() + assert(cond_status is True) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + return workflow def print_workflow(self, workflow): @@ -458,7 +534,13 @@ def test_workflow(self): cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + # multiple conditions + # cond8 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond8 = Condition(cond=work1.is_finished) + cond9 = CompositeCondition(conditions=[work4.is_finished, cond8.is_condition_true], true_works=[work6], false_works=work7) + workflow.add_condition(cond7) + workflow.add_condition(cond9) id_works = workflow.independent_works # print(id_works) id_works.sort() @@ -553,6 +635,82 @@ def test_workflow(self): work5.status = WorkStatus.New work1.status = WorkStatus.New + # multiple conditions + # cond8 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + # cond8 = Condition(cond=work1.is_finished) + # cond9 = CompositeCondition(conditions=[work4.is_finished, cond8.is_condition_true], true_works=[work6], false_works=work7) + + works = cond9.all_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4, work6, work7]) + works = cond9.all_pre_works() + works.sort(key=lambda x: x.work_id) + assert(works == [work1, work4]) + works = cond9.all_next_works() + works.sort(key=lambda x: x.work_id) + # print([w.work_id for w in works]) + assert(works == [work6, work7]) + cond_status = cond9.get_condition_status() + assert(cond_status is False) + + work4.status = WorkStatus.Finished + cond_status = cond9.get_condition_status() + assert(cond_status is False) + work1.status = WorkStatus.Finished + cond_status = cond9.get_condition_status() + assert(cond_status is True) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.NotTriggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.NotTriggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + return workflow def test_workflow_condition_reload(self): @@ -589,7 +747,13 @@ def test_workflow_condition_reload(self): cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) cond7 = CompositeCondition(conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) + # multiple conditions + # cond8 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) + cond8 = Condition(cond=work1.is_finished) + cond9 = CompositeCondition(conditions=[work4.is_finished, cond8.is_condition_true], true_works=[work6], false_works=work7) + workflow.add_condition(cond7) + workflow.add_condition(cond9) workflow_str = json_dumps(workflow, sort_keys=True, indent=4) # print(workflow_str) @@ -639,6 +803,42 @@ def test_workflow_condition_reload(self): work5.status = WorkStatus.New work1.status = WorkStatus.New + # cond9 + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == [work7]) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + assert(works == []) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + works = cond9.get_next_works(trigger=ConditionTrigger.ToTrigger) + works.sort(key=lambda x: x.work_id) + assert(works == []) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + assert(works == [work7]) + work4.status = WorkStatus.Finished + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [work6]) + work1.status = WorkStatus.Finished + works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) + works.sort(key=lambda x: x.work_id) + assert(works == [ work6]) + work4.status = WorkStatus.New + work1.status = WorkStatus.New + return workflow def test_workflow_loop(self): From 26a849055148f65dde6582c0b683e470cbfa5d91 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:20:16 +0100 Subject: [PATCH 114/156] to support class attributes as condtion --- common/lib/idds/common/dict_class.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/lib/idds/common/dict_class.py b/common/lib/idds/common/dict_class.py index bbceacd0..bdf1612f 100644 --- a/common/lib/idds/common/dict_class.py +++ b/common/lib/idds/common/dict_class.py @@ -68,6 +68,12 @@ def is_class_method(d): return True return False + @staticmethod + def is_class_attribute(d): + if d and isinstance(d, dict) and 'idds_attribute' in d and 'idds_method_class_id' in d: + return True + return False + @staticmethod def load_instance(d): module = __import__(d['module'], fromlist=[None]) @@ -83,6 +89,11 @@ def load_instance_method(d): # not do anything. Will load the method in Workflow class. return d + @staticmethod + def load_instance_attribute(d): + # not do anything. Will load the method in Workflow class. + return d + @staticmethod def from_dict(d): if not d: @@ -100,6 +111,9 @@ def from_dict(d): elif DictClass.is_class_method(d): impl = DictClass.load_instance_method(d) return impl + elif DictClass.is_class_attribute(d): + impl = DictClass.load_instance_attribute(d) + return impl elif isinstance(d, dict): for k, v in d.items(): d[k] = DictClass.from_dict(v) From cee989b59bd3c2ac4ca2e1a57679c3b5023baa3b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:27:30 +0100 Subject: [PATCH 115/156] fix global parameters --- main/lib/idds/tests/test_workflow_condition_v2.py | 2 +- main/lib/idds/tests/trigger_release.py | 2 +- monitor/conf.js | 12 ++++++------ workflow/lib/idds/workflowv2/workflow.py | 8 ++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/main/lib/idds/tests/test_workflow_condition_v2.py b/main/lib/idds/tests/test_workflow_condition_v2.py index a6007b84..3b28cac2 100644 --- a/main/lib/idds/tests/test_workflow_condition_v2.py +++ b/main/lib/idds/tests/test_workflow_condition_v2.py @@ -835,7 +835,7 @@ def test_workflow_condition_reload(self): work1.status = WorkStatus.Finished works = cond9.get_next_works(trigger=ConditionTrigger.Triggered) works.sort(key=lambda x: x.work_id) - assert(works == [ work6]) + assert(works == [work6]) work4.status = WorkStatus.New work1.status = WorkStatus.New diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index f21db22a..b5d74c34 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -12,7 +12,7 @@ request_ids = [368, 369, 370, 371, 372, 373, 374, 375, 376] -request_ids = [419] +request_ids = [475] for request_id in request_ids: contents = get_contents(request_id=request_id, status=ContentStatus.Available) ret_contents = {} diff --git a/monitor/conf.js b/monitor/conf.js index 5994e771..fe429e62 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus753.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus753.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus753.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus753.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus753.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus753.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus781.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus781.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus781.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus781.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus781.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus781.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index cb3d318c..398df75a 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -1857,6 +1857,14 @@ def refresh_parameter_links(self): if self.runs: self.runs[str(self.num_run)].refresh_parameter_links() + def set_global_parameters(self, value): + self.template.set_global_parameters(value) + + def sync_global_parameters_from_work(self, work): + if self.runs: + return self.runs[str(self.num_run)].sync_global_parameters_from_work(work) + return self.template.sync_global_parameters_from_work(work) + def get_new_works(self): self.sync_works() if self.runs: From e47bcfb55acd93354c2d31ec9f5a9bfd67205d70 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:28:53 +0100 Subject: [PATCH 116/156] new version 0.8.3 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 3085dbb1..94a2f2da 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index ce840291..76d76a5c 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.2 - - idds-workflow==0.8.2 \ No newline at end of file + - idds-common==0.8.3 + - idds-workflow==0.8.3 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 3085dbb1..94a2f2da 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 36c4a2c5..785ac9db 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.2 - - idds-workflow==0.8.2 \ No newline at end of file + - idds-common==0.8.3 + - idds-workflow==0.8.3 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 3085dbb1..94a2f2da 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 11f5e173..bb072cb6 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index d2bd7066..e75bb4bc 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.2 - - idds-workflow==0.8.2 \ No newline at end of file + - idds-common==0.8.3 + - idds-workflow==0.8.3 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 3085dbb1..94a2f2da 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 8640646e..38a31d8a 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.2 - - idds-workflow==0.8.2 - - idds-client==0.8.2 \ No newline at end of file + - idds-common==0.8.3 + - idds-workflow==0.8.3 + - idds-client==0.8.3 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 3085dbb1..94a2f2da 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/website/version.py b/website/version.py index 3085dbb1..94a2f2da 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 3085dbb1..94a2f2da 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.2" +release_version = "0.8.3" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 683c251d..db867f69 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.2 \ No newline at end of file + - idds-common==0.8.3 \ No newline at end of file From fba68f9263144b15515f2075d75a51857e17f2d2 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:43:44 +0100 Subject: [PATCH 117/156] add docs for global parameters --- docs/source/users/workflow_examples.rst | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/source/users/workflow_examples.rst b/docs/source/users/workflow_examples.rst index e68b2d70..1aea8415 100644 --- a/docs/source/users/workflow_examples.rst +++ b/docs/source/users/workflow_examples.rst @@ -94,3 +94,30 @@ Here is a simple example of sub loop workflow with parameter links. workflow.add_work(workflow1, initial=False) workflow.add_condition(cond2) workflow.add_parameter_link(work3, work1, p_link1) + + +Workflow with global parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here is a simple example of workflow with global parameters. When a work starts, the work will use the global parameters and call 'setattr' to set the attributes for this work. When a work terminates, idds will call getattr to get the values for global parameters and store them in the global parameters. +However, to avoid the global parameters overwrite the work's private attributes, currently only parameters start with 'user_' will be accepted as global parameters. Other parameters will be ignored with a warning logging messages. + +.. code-block:: python + + from idds.workflowv2.work import Work, WorkStatus + from idds.workflowv2.workflow import (CompositeCondition, AndCondition, OrCondition, + Condition, ConditionTrigger, Workflow, ParameterLink) + + work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_1'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_1'}) + work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2, + primary_input_collection={'scope': 'test_scop', 'name': 'input_test_work_2'}, + primary_output_collection={'scope': 'test_scop', 'name': 'output_test_work_2'}) + + workflow1 = Workflow() + workflow1.add_work(work1, initial=False) + workflow1.add_work(work2, initial=False) + + # to avoid + workflow1.set_global_parameters({'user_attr1': 1, 'user_attr2': 2}) From eb92f97241bf260d3e004da9b714c0b1e71908b1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 16:44:10 +0100 Subject: [PATCH 118/156] new version 0.8.4 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 94a2f2da..0ef19cd5 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 76d76a5c..0d779dd8 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.3 - - idds-workflow==0.8.3 \ No newline at end of file + - idds-common==0.8.4 + - idds-workflow==0.8.4 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 94a2f2da..0ef19cd5 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 785ac9db..9b818272 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.3 - - idds-workflow==0.8.3 \ No newline at end of file + - idds-common==0.8.4 + - idds-workflow==0.8.4 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 94a2f2da..0ef19cd5 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index bb072cb6..642205d0 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index e75bb4bc..d932e757 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.3 - - idds-workflow==0.8.3 \ No newline at end of file + - idds-common==0.8.4 + - idds-workflow==0.8.4 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 94a2f2da..0ef19cd5 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 38a31d8a..fde1f914 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.3 - - idds-workflow==0.8.3 - - idds-client==0.8.3 \ No newline at end of file + - idds-common==0.8.4 + - idds-workflow==0.8.4 + - idds-client==0.8.4 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 94a2f2da..0ef19cd5 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/website/version.py b/website/version.py index 94a2f2da..0ef19cd5 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 94a2f2da..0ef19cd5 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.3" +release_version = "0.8.4" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index db867f69..d1e30ba3 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.3 \ No newline at end of file + - idds-common==0.8.4 \ No newline at end of file From b148474c39a74940799f78679743784deeb6f360 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 17:22:37 +0100 Subject: [PATCH 119/156] fix docs --- docs/source/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5b603ccf..f26decbd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -32,7 +32,6 @@ For documentation specific to any of these three, please see the subsequent sect general/v2/architecture general/v2/workflow general/v2/dag - general/v1/index Use Cases ========= From f096079e5e2b5ca0cc07c22bdb424dec870fb562 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 17:38:42 +0100 Subject: [PATCH 120/156] update docs sources --- docs/source/codes/atlas/idds.atlas.rst | 2 + docs/source/codes/atlas/idds.atlas.rucio.rst | 8 + docs/source/codes/atlas/modules.rst | 4 +- docs/source/codes/client/idds.client.rst | 24 +++ docs/source/codes/client/modules.rst | 4 +- docs/source/codes/common/idds.common.rst | 16 ++ docs/source/codes/common/modules.rst | 4 +- .../codes/main/idds.agents.conductor.rst | 8 + docs/source/codes/main/idds.agents.rst | 1 + .../codes/main/idds.agents.transformer.rst | 8 + docs/source/codes/main/idds.core.rst | 16 ++ docs/source/codes/main/idds.orm.rst | 16 ++ docs/source/codes/main/idds.rest.v1.rst | 32 +++ docs/source/codes/main/idds.tests.rst | 186 +++++++++++++++++- docs/source/codes/main/modules.rst | 4 +- main/tools/env/install_idds_full.sh | 11 +- 16 files changed, 326 insertions(+), 18 deletions(-) diff --git a/docs/source/codes/atlas/idds.atlas.rst b/docs/source/codes/atlas/idds.atlas.rst index 05c2523f..716b2f5e 100644 --- a/docs/source/codes/atlas/idds.atlas.rst +++ b/docs/source/codes/atlas/idds.atlas.rst @@ -11,6 +11,8 @@ Subpackages idds.atlas.processing idds.atlas.rucio idds.atlas.transformer + idds.atlas.workflow + idds.atlas.workflowv2 Submodules ---------- diff --git a/docs/source/codes/atlas/idds.atlas.rucio.rst b/docs/source/codes/atlas/idds.atlas.rucio.rst index 1c5070c8..abb75cae 100644 --- a/docs/source/codes/atlas/idds.atlas.rucio.rst +++ b/docs/source/codes/atlas/idds.atlas.rucio.rst @@ -44,6 +44,14 @@ idds.atlas.rucio.contents\_register module :undoc-members: :show-inheritance: +idds.atlas.rucio.rule\_creator module +------------------------------------- + +.. automodule:: idds.atlas.rucio.rule_creator + :members: + :undoc-members: + :show-inheritance: + idds.atlas.rucio.rule\_poller module ------------------------------------ diff --git a/docs/source/codes/atlas/modules.rst b/docs/source/codes/atlas/modules.rst index 89e4d93b..bc985f17 100644 --- a/docs/source/codes/atlas/modules.rst +++ b/docs/source/codes/atlas/modules.rst @@ -1,5 +1,5 @@ -idds atlas -========== +idds +==== .. toctree:: :maxdepth: 4 diff --git a/docs/source/codes/client/idds.client.rst b/docs/source/codes/client/idds.client.rst index ecc4ff8f..30f3e9b4 100644 --- a/docs/source/codes/client/idds.client.rst +++ b/docs/source/codes/client/idds.client.rst @@ -36,6 +36,14 @@ idds.client.client module :undoc-members: :show-inheritance: +idds.client.clientmanager module +-------------------------------- + +.. automodule:: idds.client.clientmanager + :members: + :undoc-members: + :show-inheritance: + idds.client.hpoclient module ---------------------------- @@ -44,6 +52,22 @@ idds.client.hpoclient module :undoc-members: :show-inheritance: +idds.client.logsclient module +----------------------------- + +.. automodule:: idds.client.logsclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.messageclient module +-------------------------------- + +.. automodule:: idds.client.messageclient + :members: + :undoc-members: + :show-inheritance: + idds.client.requestclient module -------------------------------- diff --git a/docs/source/codes/client/modules.rst b/docs/source/codes/client/modules.rst index 807cbd1c..bc985f17 100644 --- a/docs/source/codes/client/modules.rst +++ b/docs/source/codes/client/modules.rst @@ -1,5 +1,5 @@ -idds client -=========== +idds +==== .. toctree:: :maxdepth: 4 diff --git a/docs/source/codes/common/idds.common.rst b/docs/source/codes/common/idds.common.rst index 3353f8c4..f04f58cf 100644 --- a/docs/source/codes/common/idds.common.rst +++ b/docs/source/codes/common/idds.common.rst @@ -28,6 +28,14 @@ idds.common.constants module :undoc-members: :show-inheritance: +idds.common.dict\_class module +------------------------------ + +.. automodule:: idds.common.dict_class + :members: + :undoc-members: + :show-inheritance: + idds.common.exceptions module ----------------------------- @@ -36,6 +44,14 @@ idds.common.exceptions module :undoc-members: :show-inheritance: +idds.common.status\_utils module +-------------------------------- + +.. automodule:: idds.common.status_utils + :members: + :undoc-members: + :show-inheritance: + idds.common.utils module ------------------------ diff --git a/docs/source/codes/common/modules.rst b/docs/source/codes/common/modules.rst index 634357e5..bc985f17 100644 --- a/docs/source/codes/common/modules.rst +++ b/docs/source/codes/common/modules.rst @@ -1,5 +1,5 @@ -idds common -=========== +idds +==== .. toctree:: :maxdepth: 4 diff --git a/docs/source/codes/main/idds.agents.conductor.rst b/docs/source/codes/main/idds.agents.conductor.rst index 66461c2e..cea25f4b 100644 --- a/docs/source/codes/main/idds.agents.conductor.rst +++ b/docs/source/codes/main/idds.agents.conductor.rst @@ -12,6 +12,14 @@ idds.agents.conductor.conductor module :undoc-members: :show-inheritance: +idds.agents.conductor.consumer module +------------------------------------- + +.. automodule:: idds.agents.conductor.consumer + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/codes/main/idds.agents.rst b/docs/source/codes/main/idds.agents.rst index f96fa983..4935c60c 100644 --- a/docs/source/codes/main/idds.agents.rst +++ b/docs/source/codes/main/idds.agents.rst @@ -11,6 +11,7 @@ Subpackages idds.agents.clerk idds.agents.common idds.agents.conductor + idds.agents.marshaller idds.agents.transformer idds.agents.transporter diff --git a/docs/source/codes/main/idds.agents.transformer.rst b/docs/source/codes/main/idds.agents.transformer.rst index 043639f0..cc5746a5 100644 --- a/docs/source/codes/main/idds.agents.transformer.rst +++ b/docs/source/codes/main/idds.agents.transformer.rst @@ -4,6 +4,14 @@ idds.agents.transformer package Submodules ---------- +idds.agents.transformer.helper module +------------------------------------- + +.. automodule:: idds.agents.transformer.helper + :members: + :undoc-members: + :show-inheritance: + idds.agents.transformer.transformer module ------------------------------------------ diff --git a/docs/source/codes/main/idds.core.rst b/docs/source/codes/main/idds.core.rst index f20a5f23..52b144f0 100644 --- a/docs/source/codes/main/idds.core.rst +++ b/docs/source/codes/main/idds.core.rst @@ -12,6 +12,14 @@ idds.core.catalog module :undoc-members: :show-inheritance: +idds.core.health module +----------------------- + +.. automodule:: idds.core.health + :members: + :undoc-members: + :show-inheritance: + idds.core.messages module ------------------------- @@ -44,6 +52,14 @@ idds.core.transforms module :undoc-members: :show-inheritance: +idds.core.workprogress module +----------------------------- + +.. automodule:: idds.core.workprogress + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/codes/main/idds.orm.rst b/docs/source/codes/main/idds.orm.rst index 4ed3bdfc..7c91f641 100644 --- a/docs/source/codes/main/idds.orm.rst +++ b/docs/source/codes/main/idds.orm.rst @@ -28,6 +28,14 @@ idds.orm.contents module :undoc-members: :show-inheritance: +idds.orm.health module +---------------------- + +.. automodule:: idds.orm.health + :members: + :undoc-members: + :show-inheritance: + idds.orm.messages module ------------------------ @@ -60,6 +68,14 @@ idds.orm.transforms module :undoc-members: :show-inheritance: +idds.orm.workprogress module +---------------------------- + +.. automodule:: idds.orm.workprogress + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/codes/main/idds.rest.v1.rst b/docs/source/codes/main/idds.rest.v1.rst index 6bcc5a4b..251a4d88 100644 --- a/docs/source/codes/main/idds.rest.v1.rst +++ b/docs/source/codes/main/idds.rest.v1.rst @@ -44,6 +44,30 @@ idds.rest.v1.hyperparameteropt module :undoc-members: :show-inheritance: +idds.rest.v1.logs module +------------------------ + +.. automodule:: idds.rest.v1.logs + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.messages module +---------------------------- + +.. automodule:: idds.rest.v1.messages + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.monitor module +--------------------------- + +.. automodule:: idds.rest.v1.monitor + :members: + :undoc-members: + :show-inheritance: + idds.rest.v1.requests module ---------------------------- @@ -52,6 +76,14 @@ idds.rest.v1.requests module :undoc-members: :show-inheritance: +idds.rest.v1.utils module +------------------------- + +.. automodule:: idds.rest.v1.utils + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/codes/main/idds.tests.rst b/docs/source/codes/main/idds.tests.rst index ef9d171e..0bb78901 100644 --- a/docs/source/codes/main/idds.tests.rst +++ b/docs/source/codes/main/idds.tests.rst @@ -12,18 +12,18 @@ idds.tests.activelearning\_test module :undoc-members: :show-inheritance: -idds.tests.broker\_test module +idds.tests.cacher\_test module ------------------------------ -.. automodule:: idds.tests.broker_test +.. automodule:: idds.tests.cacher_test :members: :undoc-members: :show-inheritance: -idds.tests.cacher\_test module ------------------------------- +idds.tests.catalog\_test module +------------------------------- -.. automodule:: idds.tests.cacher_test +.. automodule:: idds.tests.catalog_test :members: :undoc-members: :show-inheritance: @@ -44,6 +44,14 @@ idds.tests.common module :undoc-members: :show-inheritance: +idds.tests.core\_tests module +----------------------------- + +.. automodule:: idds.tests.core_tests + :members: + :undoc-members: + :show-inheritance: + idds.tests.datacarousel\_test module ------------------------------------ @@ -92,6 +100,14 @@ idds.tests.hyperparameteropt\_nevergrad\_test module :undoc-members: :show-inheritance: +idds.tests.logs\_test module +---------------------------- + +.. automodule:: idds.tests.logs_test + :members: + :undoc-members: + :show-inheritance: + idds.tests.message\_test module ------------------------------- @@ -100,6 +116,38 @@ idds.tests.message\_test module :undoc-members: :show-inheritance: +idds.tests.message\_test1 module +-------------------------------- + +.. automodule:: idds.tests.message_test1 + :members: + :undoc-members: + :show-inheritance: + +idds.tests.migrating\_requests\_v1\_to\_v2 module +------------------------------------------------- + +.. automodule:: idds.tests.migrating_requests_v1_to_v2 + :members: + :undoc-members: + :show-inheritance: + +idds.tests.panda\_iam\_test module +---------------------------------- + +.. automodule:: idds.tests.panda_iam_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.panda\_test module +----------------------------- + +.. automodule:: idds.tests.panda_test + :members: + :undoc-members: + :show-inheritance: + idds.tests.performance\_test\_with\_cx\_oracle module ----------------------------------------------------- @@ -124,6 +172,38 @@ idds.tests.rest\_test module :undoc-members: :show-inheritance: +idds.tests.run\_sql module +-------------------------- + +.. automodule:: idds.tests.run_sql + :members: + :undoc-members: + :show-inheritance: + +idds.tests.scaling\_checks module +--------------------------------- + +.. automodule:: idds.tests.scaling_checks + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_activelearning module +-------------------------------------- + +.. automodule:: idds.tests.test_activelearning + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_atlaspandawork module +-------------------------------------- + +.. automodule:: idds.tests.test_atlaspandawork + :members: + :undoc-members: + :show-inheritance: + idds.tests.test\_catalog module ------------------------------- @@ -132,6 +212,54 @@ idds.tests.test\_catalog module :undoc-members: :show-inheritance: +idds.tests.test\_datacarousel module +------------------------------------ + +.. automodule:: idds.tests.test_datacarousel + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_domapanda module +--------------------------------- + +.. automodule:: idds.tests.test_domapanda + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_domapanda\_workflow module +------------------------------------------- + +.. automodule:: idds.tests.test_domapanda_workflow + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_hyperparameteropt module +----------------------------------------- + +.. automodule:: idds.tests.test_hyperparameteropt + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_migrate\_requests module +----------------------------------------- + +.. automodule:: idds.tests.test_migrate_requests + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_property module +-------------------------------- + +.. automodule:: idds.tests.test_property + :members: + :undoc-members: + :show-inheritance: + idds.tests.test\_request\_transform module ------------------------------------------ @@ -148,6 +276,22 @@ idds.tests.test\_requests module :undoc-members: :show-inheritance: +idds.tests.test\_running\_data module +------------------------------------- + +.. automodule:: idds.tests.test_running_data + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_scaling module +------------------------------- + +.. automodule:: idds.tests.test_scaling + :members: + :undoc-members: + :show-inheritance: + idds.tests.test\_transform\_collection\_content module ------------------------------------------------------ @@ -164,6 +308,38 @@ idds.tests.test\_transform\_processing module :undoc-members: :show-inheritance: +idds.tests.test\_workflow module +-------------------------------- + +.. automodule:: idds.tests.test_workflow + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_workflow\_condition module +------------------------------------------- + +.. automodule:: idds.tests.test_workflow_condition + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_workflow\_condition\_v2 module +----------------------------------------------- + +.. automodule:: idds.tests.test_workflow_condition_v2 + :members: + :undoc-members: + :show-inheritance: + +idds.tests.trigger\_release module +---------------------------------- + +.. automodule:: idds.tests.trigger_release + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/codes/main/modules.rst b/docs/source/codes/main/modules.rst index 37de9e1c..bc985f17 100644 --- a/docs/source/codes/main/modules.rst +++ b/docs/source/codes/main/modules.rst @@ -1,5 +1,5 @@ -idds main -========= +idds +==== .. toctree:: :maxdepth: 4 diff --git a/main/tools/env/install_idds_full.sh b/main/tools/env/install_idds_full.sh index 8ee76f1c..35c10cdc 100644 --- a/main/tools/env/install_idds_full.sh +++ b/main/tools/env/install_idds_full.sh @@ -75,8 +75,9 @@ pip install --upgrade sphinx-rtd-theme sphinx-quickstart make clean make html -sphinx-apidoc -o ./source/codes/main/ ../main/lib/idds -sphinx-apidoc -o ./source/codes/common/ ../common/lib/idds -sphinx-apidoc -o ./source/codes/client/ ../client/lib/idds -sphinx-apidoc -o ./source/codes/atlas/ ../atlas/lib/idds - +sphinx-apidoc -f -o ./source/codes/main/ ../main/lib/idds +sphinx-apidoc -f -o ./source/codes/common/ ../common/lib/idds +sphinx-apidoc -f -o ./source/codes/client/ ../client/lib/idds +sphinx-apidoc -f -o ./source/codes/workflow/ ../workflow/lib/idds +sphinx-apidoc -f -o ./source/codes/atlas/ ../atlas/lib/idds +sphinx-apidoc -f -o ./source/codes/doma/ ../doma/lib/idds From 8b472a4cd0bfa7f5545bee913d8e8b5592151019 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 17:45:52 +0100 Subject: [PATCH 121/156] update docs sources --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index f26decbd..575d1c5f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -66,7 +66,7 @@ Source Codes .. toctree:: :maxdepth: 1 - codes/libraries + .. codes/libraries Indices and tables ================== From 84aaec4fdd0dff5361a9f39d23838f2c02142d01 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 17:50:12 +0100 Subject: [PATCH 122/156] remove source codes doces --- .../codes/atlas/idds.atlas.notifier.rst | 22 -- .../codes/atlas/idds.atlas.processing.rst | 102 ----- docs/source/codes/atlas/idds.atlas.rst | 35 -- docs/source/codes/atlas/idds.atlas.rucio.rst | 78 ---- .../codes/atlas/idds.atlas.transformer.rst | 46 --- docs/source/codes/atlas/idds.rst | 18 - docs/source/codes/atlas/modules.rst | 7 - docs/source/codes/client/idds.client.rst | 94 ----- docs/source/codes/client/idds.rst | 18 - docs/source/codes/client/modules.rst | 7 - .../codes/common/idds.common.plugin.rst | 30 -- docs/source/codes/common/idds.common.rst | 78 ---- docs/source/codes/common/idds.rst | 18 - docs/source/codes/common/modules.rst | 7 - docs/source/codes/libraries.rst | 44 --- .../source/codes/main/idds.agents.carrier.rst | 22 -- docs/source/codes/main/idds.agents.clerk.rst | 22 -- docs/source/codes/main/idds.agents.common.rst | 38 -- .../codes/main/idds.agents.conductor.rst | 30 -- docs/source/codes/main/idds.agents.rst | 36 -- .../codes/main/idds.agents.transformer.rst | 30 -- .../codes/main/idds.agents.transporter.rst | 22 -- docs/source/codes/main/idds.api.rst | 62 ---- docs/source/codes/main/idds.core.rst | 70 ---- docs/source/codes/main/idds.orm.base.rst | 54 --- docs/source/codes/main/idds.orm.rst | 86 ----- docs/source/codes/main/idds.rest.rst | 18 - docs/source/codes/main/idds.rest.v1.rst | 94 ----- docs/source/codes/main/idds.rst | 35 -- docs/source/codes/main/idds.tests.rst | 350 ------------------ docs/source/codes/main/modules.rst | 7 - 31 files changed, 1580 deletions(-) delete mode 100644 docs/source/codes/atlas/idds.atlas.notifier.rst delete mode 100644 docs/source/codes/atlas/idds.atlas.processing.rst delete mode 100644 docs/source/codes/atlas/idds.atlas.rst delete mode 100644 docs/source/codes/atlas/idds.atlas.rucio.rst delete mode 100644 docs/source/codes/atlas/idds.atlas.transformer.rst delete mode 100644 docs/source/codes/atlas/idds.rst delete mode 100644 docs/source/codes/atlas/modules.rst delete mode 100644 docs/source/codes/client/idds.client.rst delete mode 100644 docs/source/codes/client/idds.rst delete mode 100644 docs/source/codes/client/modules.rst delete mode 100644 docs/source/codes/common/idds.common.plugin.rst delete mode 100644 docs/source/codes/common/idds.common.rst delete mode 100644 docs/source/codes/common/idds.rst delete mode 100644 docs/source/codes/common/modules.rst delete mode 100644 docs/source/codes/libraries.rst delete mode 100644 docs/source/codes/main/idds.agents.carrier.rst delete mode 100644 docs/source/codes/main/idds.agents.clerk.rst delete mode 100644 docs/source/codes/main/idds.agents.common.rst delete mode 100644 docs/source/codes/main/idds.agents.conductor.rst delete mode 100644 docs/source/codes/main/idds.agents.rst delete mode 100644 docs/source/codes/main/idds.agents.transformer.rst delete mode 100644 docs/source/codes/main/idds.agents.transporter.rst delete mode 100644 docs/source/codes/main/idds.api.rst delete mode 100644 docs/source/codes/main/idds.core.rst delete mode 100644 docs/source/codes/main/idds.orm.base.rst delete mode 100644 docs/source/codes/main/idds.orm.rst delete mode 100644 docs/source/codes/main/idds.rest.rst delete mode 100644 docs/source/codes/main/idds.rest.v1.rst delete mode 100644 docs/source/codes/main/idds.rst delete mode 100644 docs/source/codes/main/idds.tests.rst delete mode 100644 docs/source/codes/main/modules.rst diff --git a/docs/source/codes/atlas/idds.atlas.notifier.rst b/docs/source/codes/atlas/idds.atlas.notifier.rst deleted file mode 100644 index 278b1da1..00000000 --- a/docs/source/codes/atlas/idds.atlas.notifier.rst +++ /dev/null @@ -1,22 +0,0 @@ -idds.atlas.notifier package -=========================== - -Submodules ----------- - -idds.atlas.notifier.messaging module ------------------------------------- - -.. automodule:: idds.atlas.notifier.messaging - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.atlas.notifier - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.processing.rst b/docs/source/codes/atlas/idds.atlas.processing.rst deleted file mode 100644 index 94b2e267..00000000 --- a/docs/source/codes/atlas/idds.atlas.processing.rst +++ /dev/null @@ -1,102 +0,0 @@ -idds.atlas.processing package -============================= - -Submodules ----------- - -idds.atlas.processing.activelearning\_condor\_poller module ------------------------------------------------------------ - -.. automodule:: idds.atlas.processing.activelearning_condor_poller - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.activelearning\_condor\_submitter module --------------------------------------------------------------- - -.. automodule:: idds.atlas.processing.activelearning_condor_submitter - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.base\_plugin module ------------------------------------------ - -.. automodule:: idds.atlas.processing.base_plugin - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.condor\_poller module -------------------------------------------- - -.. automodule:: idds.atlas.processing.condor_poller - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.condor\_submitter module ----------------------------------------------- - -.. automodule:: idds.atlas.processing.condor_submitter - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.hyperparameteropt\_bayesian module --------------------------------------------------------- - -.. automodule:: idds.atlas.processing.hyperparameteropt_bayesian - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.hyperparameteropt\_condor\_poller module --------------------------------------------------------------- - -.. automodule:: idds.atlas.processing.hyperparameteropt_condor_poller - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.hyperparameteropt\_condor\_submitter module ------------------------------------------------------------------ - -.. automodule:: idds.atlas.processing.hyperparameteropt_condor_submitter - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.hyperparameteropt\_nevergrad module ---------------------------------------------------------- - -.. automodule:: idds.atlas.processing.hyperparameteropt_nevergrad - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.stagein\_poller module --------------------------------------------- - -.. automodule:: idds.atlas.processing.stagein_poller - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.processing.stagein\_submitter module ------------------------------------------------ - -.. automodule:: idds.atlas.processing.stagein_submitter - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.atlas.processing - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.rst b/docs/source/codes/atlas/idds.atlas.rst deleted file mode 100644 index 716b2f5e..00000000 --- a/docs/source/codes/atlas/idds.atlas.rst +++ /dev/null @@ -1,35 +0,0 @@ -idds.atlas package -================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.atlas.notifier - idds.atlas.processing - idds.atlas.rucio - idds.atlas.transformer - idds.atlas.workflow - idds.atlas.workflowv2 - -Submodules ----------- - -idds.atlas.version module -------------------------- - -.. automodule:: idds.atlas.version - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.atlas - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.rucio.rst b/docs/source/codes/atlas/idds.atlas.rucio.rst deleted file mode 100644 index abb75cae..00000000 --- a/docs/source/codes/atlas/idds.atlas.rucio.rst +++ /dev/null @@ -1,78 +0,0 @@ -idds.atlas.rucio package -======================== - -Submodules ----------- - -idds.atlas.rucio.base\_plugin module ------------------------------------- - -.. automodule:: idds.atlas.rucio.base_plugin - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.collection\_lister module ------------------------------------------- - -.. automodule:: idds.atlas.rucio.collection_lister - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.collection\_metadata\_reader module ----------------------------------------------------- - -.. automodule:: idds.atlas.rucio.collection_metadata_reader - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.contents\_lister module ----------------------------------------- - -.. automodule:: idds.atlas.rucio.contents_lister - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.contents\_register module ------------------------------------------- - -.. automodule:: idds.atlas.rucio.contents_register - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.rule\_creator module -------------------------------------- - -.. automodule:: idds.atlas.rucio.rule_creator - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.rule\_poller module ------------------------------------- - -.. automodule:: idds.atlas.rucio.rule_poller - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.rucio.rule\_submitter module ---------------------------------------- - -.. automodule:: idds.atlas.rucio.rule_submitter - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.atlas.rucio - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.transformer.rst b/docs/source/codes/atlas/idds.atlas.transformer.rst deleted file mode 100644 index 3679e6c7..00000000 --- a/docs/source/codes/atlas/idds.atlas.transformer.rst +++ /dev/null @@ -1,46 +0,0 @@ -idds.atlas.transformer package -============================== - -Submodules ----------- - -idds.atlas.transformer.activelearning\_transformer module ---------------------------------------------------------- - -.. automodule:: idds.atlas.transformer.activelearning_transformer - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.transformer.base\_plugin module ------------------------------------------- - -.. automodule:: idds.atlas.transformer.base_plugin - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.transformer.hyperparameteropt\_transformer module ------------------------------------------------------------- - -.. automodule:: idds.atlas.transformer.hyperparameteropt_transformer - :members: - :undoc-members: - :show-inheritance: - -idds.atlas.transformer.stagein\_transformer module --------------------------------------------------- - -.. automodule:: idds.atlas.transformer.stagein_transformer - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.atlas.transformer - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/atlas/idds.rst b/docs/source/codes/atlas/idds.rst deleted file mode 100644 index d942f08f..00000000 --- a/docs/source/codes/atlas/idds.rst +++ /dev/null @@ -1,18 +0,0 @@ -idds package -============ - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.atlas - -Module contents ---------------- - -.. automodule:: idds - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/atlas/modules.rst b/docs/source/codes/atlas/modules.rst deleted file mode 100644 index bc985f17..00000000 --- a/docs/source/codes/atlas/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -idds -==== - -.. toctree:: - :maxdepth: 4 - - idds diff --git a/docs/source/codes/client/idds.client.rst b/docs/source/codes/client/idds.client.rst deleted file mode 100644 index 30f3e9b4..00000000 --- a/docs/source/codes/client/idds.client.rst +++ /dev/null @@ -1,94 +0,0 @@ -idds.client package -=================== - -Submodules ----------- - -idds.client.base module ------------------------ - -.. automodule:: idds.client.base - :members: - :undoc-members: - :show-inheritance: - -idds.client.cacherclient module -------------------------------- - -.. automodule:: idds.client.cacherclient - :members: - :undoc-members: - :show-inheritance: - -idds.client.catalogclient module --------------------------------- - -.. automodule:: idds.client.catalogclient - :members: - :undoc-members: - :show-inheritance: - -idds.client.client module -------------------------- - -.. automodule:: idds.client.client - :members: - :undoc-members: - :show-inheritance: - -idds.client.clientmanager module --------------------------------- - -.. automodule:: idds.client.clientmanager - :members: - :undoc-members: - :show-inheritance: - -idds.client.hpoclient module ----------------------------- - -.. automodule:: idds.client.hpoclient - :members: - :undoc-members: - :show-inheritance: - -idds.client.logsclient module ------------------------------ - -.. automodule:: idds.client.logsclient - :members: - :undoc-members: - :show-inheritance: - -idds.client.messageclient module --------------------------------- - -.. automodule:: idds.client.messageclient - :members: - :undoc-members: - :show-inheritance: - -idds.client.requestclient module --------------------------------- - -.. automodule:: idds.client.requestclient - :members: - :undoc-members: - :show-inheritance: - -idds.client.version module --------------------------- - -.. automodule:: idds.client.version - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.client - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/client/idds.rst b/docs/source/codes/client/idds.rst deleted file mode 100644 index 872d8e5c..00000000 --- a/docs/source/codes/client/idds.rst +++ /dev/null @@ -1,18 +0,0 @@ -idds package -============ - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.client - -Module contents ---------------- - -.. automodule:: idds - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/client/modules.rst b/docs/source/codes/client/modules.rst deleted file mode 100644 index bc985f17..00000000 --- a/docs/source/codes/client/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -idds -==== - -.. toctree:: - :maxdepth: 4 - - idds diff --git a/docs/source/codes/common/idds.common.plugin.rst b/docs/source/codes/common/idds.common.plugin.rst deleted file mode 100644 index 9c9b99a7..00000000 --- a/docs/source/codes/common/idds.common.plugin.rst +++ /dev/null @@ -1,30 +0,0 @@ -idds.common.plugin package -========================== - -Submodules ----------- - -idds.common.plugin.plugin\_base module --------------------------------------- - -.. automodule:: idds.common.plugin.plugin_base - :members: - :undoc-members: - :show-inheritance: - -idds.common.plugin.plugin\_utils module ---------------------------------------- - -.. automodule:: idds.common.plugin.plugin_utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.common.plugin - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/common/idds.common.rst b/docs/source/codes/common/idds.common.rst deleted file mode 100644 index f04f58cf..00000000 --- a/docs/source/codes/common/idds.common.rst +++ /dev/null @@ -1,78 +0,0 @@ -idds.common package -=================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.common.plugin - -Submodules ----------- - -idds.common.config module -------------------------- - -.. automodule:: idds.common.config - :members: - :undoc-members: - :show-inheritance: - -idds.common.constants module ----------------------------- - -.. automodule:: idds.common.constants - :members: - :undoc-members: - :show-inheritance: - -idds.common.dict\_class module ------------------------------- - -.. automodule:: idds.common.dict_class - :members: - :undoc-members: - :show-inheritance: - -idds.common.exceptions module ------------------------------ - -.. automodule:: idds.common.exceptions - :members: - :undoc-members: - :show-inheritance: - -idds.common.status\_utils module --------------------------------- - -.. automodule:: idds.common.status_utils - :members: - :undoc-members: - :show-inheritance: - -idds.common.utils module ------------------------- - -.. automodule:: idds.common.utils - :members: - :undoc-members: - :show-inheritance: - -idds.common.version module --------------------------- - -.. automodule:: idds.common.version - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.common - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/common/idds.rst b/docs/source/codes/common/idds.rst deleted file mode 100644 index 62094b2d..00000000 --- a/docs/source/codes/common/idds.rst +++ /dev/null @@ -1,18 +0,0 @@ -idds package -============ - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.common - -Module contents ---------------- - -.. automodule:: idds - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/common/modules.rst b/docs/source/codes/common/modules.rst deleted file mode 100644 index bc985f17..00000000 --- a/docs/source/codes/common/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -idds -==== - -.. toctree:: - :maxdepth: 4 - - idds diff --git a/docs/source/codes/libraries.rst b/docs/source/codes/libraries.rst deleted file mode 100644 index 5d4ac510..00000000 --- a/docs/source/codes/libraries.rst +++ /dev/null @@ -1,44 +0,0 @@ -iDDS Codes -========== - -iDDS codes documents are automatically generated by sphinx. - -Common libraries -**************** - -Common libaries are the basic libraries which are used by all other libraries. - -.. toctree:: - :maxdepth: 1 - - common/modules - -Main libraries -**************** - -Main libaries include the core functions, RESTful service and daemon agents which run on the servers. - -.. toctree:: - :maxdepth: 1 - - main/modules - -Client libraries -**************** - -Client libraries is used for users to communicate with iDDS service. - -.. toctree:: - :maxdepth: 1 - - client/modules - -ATLAS libraries -**************** - -ATLAS libraries include plugins for ATLAS special services. - -.. toctree:: - :maxdepth: 1 - - atlas/modules diff --git a/docs/source/codes/main/idds.agents.carrier.rst b/docs/source/codes/main/idds.agents.carrier.rst deleted file mode 100644 index 37a59e7e..00000000 --- a/docs/source/codes/main/idds.agents.carrier.rst +++ /dev/null @@ -1,22 +0,0 @@ -idds.agents.carrier package -=========================== - -Submodules ----------- - -idds.agents.carrier.carrier module ----------------------------------- - -.. automodule:: idds.agents.carrier.carrier - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents.carrier - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.clerk.rst b/docs/source/codes/main/idds.agents.clerk.rst deleted file mode 100644 index 87bbec1e..00000000 --- a/docs/source/codes/main/idds.agents.clerk.rst +++ /dev/null @@ -1,22 +0,0 @@ -idds.agents.clerk package -========================= - -Submodules ----------- - -idds.agents.clerk.clerk module ------------------------------- - -.. automodule:: idds.agents.clerk.clerk - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents.clerk - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.common.rst b/docs/source/codes/main/idds.agents.common.rst deleted file mode 100644 index 8d9114e9..00000000 --- a/docs/source/codes/main/idds.agents.common.rst +++ /dev/null @@ -1,38 +0,0 @@ -idds.agents.common package -========================== - -Submodules ----------- - -idds.agents.common.baseagent module ------------------------------------ - -.. automodule:: idds.agents.common.baseagent - :members: - :undoc-members: - :show-inheritance: - -idds.agents.common.timerscheduler module ----------------------------------------- - -.. automodule:: idds.agents.common.timerscheduler - :members: - :undoc-members: - :show-inheritance: - -idds.agents.common.timertask module ------------------------------------ - -.. automodule:: idds.agents.common.timertask - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents.common - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.conductor.rst b/docs/source/codes/main/idds.agents.conductor.rst deleted file mode 100644 index cea25f4b..00000000 --- a/docs/source/codes/main/idds.agents.conductor.rst +++ /dev/null @@ -1,30 +0,0 @@ -idds.agents.conductor package -============================= - -Submodules ----------- - -idds.agents.conductor.conductor module --------------------------------------- - -.. automodule:: idds.agents.conductor.conductor - :members: - :undoc-members: - :show-inheritance: - -idds.agents.conductor.consumer module -------------------------------------- - -.. automodule:: idds.agents.conductor.consumer - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents.conductor - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.rst b/docs/source/codes/main/idds.agents.rst deleted file mode 100644 index 4935c60c..00000000 --- a/docs/source/codes/main/idds.agents.rst +++ /dev/null @@ -1,36 +0,0 @@ -idds.agents package -=================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.agents.carrier - idds.agents.clerk - idds.agents.common - idds.agents.conductor - idds.agents.marshaller - idds.agents.transformer - idds.agents.transporter - -Submodules ----------- - -idds.agents.main module ------------------------ - -.. automodule:: idds.agents.main - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.transformer.rst b/docs/source/codes/main/idds.agents.transformer.rst deleted file mode 100644 index cc5746a5..00000000 --- a/docs/source/codes/main/idds.agents.transformer.rst +++ /dev/null @@ -1,30 +0,0 @@ -idds.agents.transformer package -=============================== - -Submodules ----------- - -idds.agents.transformer.helper module -------------------------------------- - -.. automodule:: idds.agents.transformer.helper - :members: - :undoc-members: - :show-inheritance: - -idds.agents.transformer.transformer module ------------------------------------------- - -.. automodule:: idds.agents.transformer.transformer - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents.transformer - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.transporter.rst b/docs/source/codes/main/idds.agents.transporter.rst deleted file mode 100644 index 5f4e9653..00000000 --- a/docs/source/codes/main/idds.agents.transporter.rst +++ /dev/null @@ -1,22 +0,0 @@ -idds.agents.transporter package -=============================== - -Submodules ----------- - -idds.agents.transporter.transporter module ------------------------------------------- - -.. automodule:: idds.agents.transporter.transporter - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.agents.transporter - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.api.rst b/docs/source/codes/main/idds.api.rst deleted file mode 100644 index f7c99753..00000000 --- a/docs/source/codes/main/idds.api.rst +++ /dev/null @@ -1,62 +0,0 @@ -idds.api package -================ - -Submodules ----------- - -idds.api.catalog module ------------------------ - -.. automodule:: idds.api.catalog - :members: - :undoc-members: - :show-inheritance: - -idds.api.collections module ---------------------------- - -.. automodule:: idds.api.collections - :members: - :undoc-members: - :show-inheritance: - -idds.api.contents module ------------------------- - -.. automodule:: idds.api.contents - :members: - :undoc-members: - :show-inheritance: - -idds.api.processings module ---------------------------- - -.. automodule:: idds.api.processings - :members: - :undoc-members: - :show-inheritance: - -idds.api.requests module ------------------------- - -.. automodule:: idds.api.requests - :members: - :undoc-members: - :show-inheritance: - -idds.api.transforms module --------------------------- - -.. automodule:: idds.api.transforms - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.api - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.core.rst b/docs/source/codes/main/idds.core.rst deleted file mode 100644 index 52b144f0..00000000 --- a/docs/source/codes/main/idds.core.rst +++ /dev/null @@ -1,70 +0,0 @@ -idds.core package -================= - -Submodules ----------- - -idds.core.catalog module ------------------------- - -.. automodule:: idds.core.catalog - :members: - :undoc-members: - :show-inheritance: - -idds.core.health module ------------------------ - -.. automodule:: idds.core.health - :members: - :undoc-members: - :show-inheritance: - -idds.core.messages module -------------------------- - -.. automodule:: idds.core.messages - :members: - :undoc-members: - :show-inheritance: - -idds.core.processings module ----------------------------- - -.. automodule:: idds.core.processings - :members: - :undoc-members: - :show-inheritance: - -idds.core.requests module -------------------------- - -.. automodule:: idds.core.requests - :members: - :undoc-members: - :show-inheritance: - -idds.core.transforms module ---------------------------- - -.. automodule:: idds.core.transforms - :members: - :undoc-members: - :show-inheritance: - -idds.core.workprogress module ------------------------------ - -.. automodule:: idds.core.workprogress - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.core - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.orm.base.rst b/docs/source/codes/main/idds.orm.base.rst deleted file mode 100644 index 8bbc9c10..00000000 --- a/docs/source/codes/main/idds.orm.base.rst +++ /dev/null @@ -1,54 +0,0 @@ -idds.orm.base package -===================== - -Submodules ----------- - -idds.orm.base.enum module -------------------------- - -.. automodule:: idds.orm.base.enum - :members: - :undoc-members: - :show-inheritance: - -idds.orm.base.models module ---------------------------- - -.. automodule:: idds.orm.base.models - :members: - :undoc-members: - :show-inheritance: - -idds.orm.base.session module ----------------------------- - -.. automodule:: idds.orm.base.session - :members: - :undoc-members: - :show-inheritance: - -idds.orm.base.types module --------------------------- - -.. automodule:: idds.orm.base.types - :members: - :undoc-members: - :show-inheritance: - -idds.orm.base.utils module --------------------------- - -.. automodule:: idds.orm.base.utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.orm.base - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.orm.rst b/docs/source/codes/main/idds.orm.rst deleted file mode 100644 index 7c91f641..00000000 --- a/docs/source/codes/main/idds.orm.rst +++ /dev/null @@ -1,86 +0,0 @@ -idds.orm package -================ - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.orm.base - -Submodules ----------- - -idds.orm.collections module ---------------------------- - -.. automodule:: idds.orm.collections - :members: - :undoc-members: - :show-inheritance: - -idds.orm.contents module ------------------------- - -.. automodule:: idds.orm.contents - :members: - :undoc-members: - :show-inheritance: - -idds.orm.health module ----------------------- - -.. automodule:: idds.orm.health - :members: - :undoc-members: - :show-inheritance: - -idds.orm.messages module ------------------------- - -.. automodule:: idds.orm.messages - :members: - :undoc-members: - :show-inheritance: - -idds.orm.processings module ---------------------------- - -.. automodule:: idds.orm.processings - :members: - :undoc-members: - :show-inheritance: - -idds.orm.requests module ------------------------- - -.. automodule:: idds.orm.requests - :members: - :undoc-members: - :show-inheritance: - -idds.orm.transforms module --------------------------- - -.. automodule:: idds.orm.transforms - :members: - :undoc-members: - :show-inheritance: - -idds.orm.workprogress module ----------------------------- - -.. automodule:: idds.orm.workprogress - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.orm - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.rest.rst b/docs/source/codes/main/idds.rest.rst deleted file mode 100644 index 72a7f882..00000000 --- a/docs/source/codes/main/idds.rest.rst +++ /dev/null @@ -1,18 +0,0 @@ -idds.rest package -================= - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.rest.v1 - -Module contents ---------------- - -.. automodule:: idds.rest - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.rest.v1.rst b/docs/source/codes/main/idds.rest.v1.rst deleted file mode 100644 index 251a4d88..00000000 --- a/docs/source/codes/main/idds.rest.v1.rst +++ /dev/null @@ -1,94 +0,0 @@ -idds.rest.v1 package -==================== - -Submodules ----------- - -idds.rest.v1.app module ------------------------ - -.. automodule:: idds.rest.v1.app - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.cacher module --------------------------- - -.. automodule:: idds.rest.v1.cacher - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.catalog module ---------------------------- - -.. automodule:: idds.rest.v1.catalog - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.controller module ------------------------------- - -.. automodule:: idds.rest.v1.controller - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.hyperparameteropt module -------------------------------------- - -.. automodule:: idds.rest.v1.hyperparameteropt - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.logs module ------------------------- - -.. automodule:: idds.rest.v1.logs - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.messages module ----------------------------- - -.. automodule:: idds.rest.v1.messages - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.monitor module ---------------------------- - -.. automodule:: idds.rest.v1.monitor - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.requests module ----------------------------- - -.. automodule:: idds.rest.v1.requests - :members: - :undoc-members: - :show-inheritance: - -idds.rest.v1.utils module -------------------------- - -.. automodule:: idds.rest.v1.utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.rest.v1 - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.rst b/docs/source/codes/main/idds.rst deleted file mode 100644 index ed271e11..00000000 --- a/docs/source/codes/main/idds.rst +++ /dev/null @@ -1,35 +0,0 @@ -idds package -============ - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - idds.agents - idds.api - idds.core - idds.orm - idds.rest - idds.tests - -Submodules ----------- - -idds.version module -------------------- - -.. automodule:: idds.version - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/idds.tests.rst b/docs/source/codes/main/idds.tests.rst deleted file mode 100644 index 0bb78901..00000000 --- a/docs/source/codes/main/idds.tests.rst +++ /dev/null @@ -1,350 +0,0 @@ -idds.tests package -================== - -Submodules ----------- - -idds.tests.activelearning\_test module --------------------------------------- - -.. automodule:: idds.tests.activelearning_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.cacher\_test module ------------------------------- - -.. automodule:: idds.tests.cacher_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.catalog\_test module -------------------------------- - -.. automodule:: idds.tests.catalog_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.client\_test module ------------------------------- - -.. automodule:: idds.tests.client_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.common module ------------------------- - -.. automodule:: idds.tests.common - :members: - :undoc-members: - :show-inheritance: - -idds.tests.core\_tests module ------------------------------ - -.. automodule:: idds.tests.core_tests - :members: - :undoc-members: - :show-inheritance: - -idds.tests.datacarousel\_test module ------------------------------------- - -.. automodule:: idds.tests.datacarousel_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.hyperparameteropt\_bayesian\_test module ---------------------------------------------------- - -.. automodule:: idds.tests.hyperparameteropt_bayesian_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.hyperparameteropt\_client\_test module -------------------------------------------------- - -.. automodule:: idds.tests.hyperparameteropt_client_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.hyperparameteropt\_docker\_local\_test module --------------------------------------------------------- - -.. automodule:: idds.tests.hyperparameteropt_docker_local_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.hyperparameteropt\_docker\_test module -------------------------------------------------- - -.. automodule:: idds.tests.hyperparameteropt_docker_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.hyperparameteropt\_nevergrad\_test module ----------------------------------------------------- - -.. automodule:: idds.tests.hyperparameteropt_nevergrad_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.logs\_test module ----------------------------- - -.. automodule:: idds.tests.logs_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.message\_test module -------------------------------- - -.. automodule:: idds.tests.message_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.message\_test1 module --------------------------------- - -.. automodule:: idds.tests.message_test1 - :members: - :undoc-members: - :show-inheritance: - -idds.tests.migrating\_requests\_v1\_to\_v2 module -------------------------------------------------- - -.. automodule:: idds.tests.migrating_requests_v1_to_v2 - :members: - :undoc-members: - :show-inheritance: - -idds.tests.panda\_iam\_test module ----------------------------------- - -.. automodule:: idds.tests.panda_iam_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.panda\_test module ------------------------------ - -.. automodule:: idds.tests.panda_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.performance\_test\_with\_cx\_oracle module ------------------------------------------------------ - -.. automodule:: idds.tests.performance_test_with_cx_oracle - :members: - :undoc-members: - :show-inheritance: - -idds.tests.performance\_test\_with\_sqlalchemy module ------------------------------------------------------ - -.. automodule:: idds.tests.performance_test_with_sqlalchemy - :members: - :undoc-members: - :show-inheritance: - -idds.tests.rest\_test module ----------------------------- - -.. automodule:: idds.tests.rest_test - :members: - :undoc-members: - :show-inheritance: - -idds.tests.run\_sql module --------------------------- - -.. automodule:: idds.tests.run_sql - :members: - :undoc-members: - :show-inheritance: - -idds.tests.scaling\_checks module ---------------------------------- - -.. automodule:: idds.tests.scaling_checks - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_activelearning module --------------------------------------- - -.. automodule:: idds.tests.test_activelearning - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_atlaspandawork module --------------------------------------- - -.. automodule:: idds.tests.test_atlaspandawork - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_catalog module -------------------------------- - -.. automodule:: idds.tests.test_catalog - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_datacarousel module ------------------------------------- - -.. automodule:: idds.tests.test_datacarousel - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_domapanda module ---------------------------------- - -.. automodule:: idds.tests.test_domapanda - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_domapanda\_workflow module -------------------------------------------- - -.. automodule:: idds.tests.test_domapanda_workflow - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_hyperparameteropt module ------------------------------------------ - -.. automodule:: idds.tests.test_hyperparameteropt - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_migrate\_requests module ------------------------------------------ - -.. automodule:: idds.tests.test_migrate_requests - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_property module --------------------------------- - -.. automodule:: idds.tests.test_property - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_request\_transform module ------------------------------------------- - -.. automodule:: idds.tests.test_request_transform - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_requests module --------------------------------- - -.. automodule:: idds.tests.test_requests - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_running\_data module -------------------------------------- - -.. automodule:: idds.tests.test_running_data - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_scaling module -------------------------------- - -.. automodule:: idds.tests.test_scaling - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_transform\_collection\_content module ------------------------------------------------------- - -.. automodule:: idds.tests.test_transform_collection_content - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_transform\_processing module ---------------------------------------------- - -.. automodule:: idds.tests.test_transform_processing - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_workflow module --------------------------------- - -.. automodule:: idds.tests.test_workflow - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_workflow\_condition module -------------------------------------------- - -.. automodule:: idds.tests.test_workflow_condition - :members: - :undoc-members: - :show-inheritance: - -idds.tests.test\_workflow\_condition\_v2 module ------------------------------------------------ - -.. automodule:: idds.tests.test_workflow_condition_v2 - :members: - :undoc-members: - :show-inheritance: - -idds.tests.trigger\_release module ----------------------------------- - -.. automodule:: idds.tests.trigger_release - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: idds.tests - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/codes/main/modules.rst b/docs/source/codes/main/modules.rst deleted file mode 100644 index bc985f17..00000000 --- a/docs/source/codes/main/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -idds -==== - -.. toctree:: - :maxdepth: 4 - - idds From d257d9b24c807c1bbfd0d3cf4bd3ce3aa6fe0099 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 17:52:27 +0100 Subject: [PATCH 123/156] remove source codes doces --- docs/source/index.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 575d1c5f..085f5cea 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,14 +60,6 @@ User Documentation users/admin_guides users/contributing -Source Codes -============= - -.. toctree:: - :maxdepth: 1 - - .. codes/libraries - Indices and tables ================== From f293ca7b3596ad5e347e9c7e885f13ff33e51dec Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 18:00:37 +0100 Subject: [PATCH 124/156] fix docs --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 39fba09d..73926bf6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -7,3 +7,4 @@ nose # nose test tools sphinx # sphinx documentation recommonmark # use Markdown with Sphinx sphinx-rtd-theme # sphinx readthedoc theme +docutils<0.18 From 3eaeac103177d68753448091207171aa28f7c072 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 18:09:13 +0100 Subject: [PATCH 125/156] fix docs --- docs/source/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 085f5cea..39d315ef 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,6 +60,14 @@ User Documentation users/admin_guides users/contributing +Source Codes +============= + +.. toctree:: + :maxdepth: 1 + + codes/libraries + Indices and tables ================== From 65b4799d06bb7cce2ba1b50d040bdcb8538afd7f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 9 Nov 2021 18:11:05 +0100 Subject: [PATCH 126/156] fix docs --- .../codes/atlas/idds.atlas.notifier.rst | 22 ++ .../codes/atlas/idds.atlas.processing.rst | 102 +++++ docs/source/codes/atlas/idds.atlas.rst | 35 ++ docs/source/codes/atlas/idds.atlas.rucio.rst | 78 ++++ .../codes/atlas/idds.atlas.transformer.rst | 46 +++ .../codes/atlas/idds.atlas.workflow.rst | 62 ++++ .../codes/atlas/idds.atlas.workflowv2.rst | 62 ++++ docs/source/codes/atlas/idds.rst | 18 + docs/source/codes/atlas/modules.rst | 7 + docs/source/codes/client/idds.client.rst | 94 +++++ docs/source/codes/client/idds.rst | 18 + docs/source/codes/client/modules.rst | 7 + .../codes/common/idds.common.plugin.rst | 30 ++ docs/source/codes/common/idds.common.rst | 78 ++++ docs/source/codes/common/idds.rst | 18 + docs/source/codes/common/modules.rst | 7 + docs/source/codes/doma/idds.doma.rst | 31 ++ docs/source/codes/doma/idds.doma.workflow.rst | 22 ++ .../codes/doma/idds.doma.workflowv2.rst | 22 ++ docs/source/codes/doma/idds.rst | 18 + docs/source/codes/doma/modules.rst | 7 + docs/source/codes/libraries.rst | 64 ++++ .../source/codes/main/idds.agents.carrier.rst | 22 ++ docs/source/codes/main/idds.agents.clerk.rst | 22 ++ docs/source/codes/main/idds.agents.common.rst | 38 ++ .../codes/main/idds.agents.conductor.rst | 30 ++ .../codes/main/idds.agents.marshaller.rst | 22 ++ docs/source/codes/main/idds.agents.rst | 36 ++ .../codes/main/idds.agents.transformer.rst | 30 ++ .../codes/main/idds.agents.transporter.rst | 22 ++ docs/source/codes/main/idds.api.rst | 62 ++++ docs/source/codes/main/idds.core.rst | 70 ++++ docs/source/codes/main/idds.orm.base.rst | 54 +++ docs/source/codes/main/idds.orm.rst | 86 +++++ docs/source/codes/main/idds.rest.rst | 18 + docs/source/codes/main/idds.rest.v1.rst | 94 +++++ docs/source/codes/main/idds.rst | 35 ++ docs/source/codes/main/idds.tests.rst | 350 ++++++++++++++++++ docs/source/codes/main/modules.rst | 7 + docs/source/codes/workflow/idds.rst | 19 + docs/source/codes/workflow/idds.workflow.rst | 62 ++++ .../source/codes/workflow/idds.workflowv2.rst | 46 +++ docs/source/codes/workflow/modules.rst | 7 + 43 files changed, 1980 insertions(+) create mode 100644 docs/source/codes/atlas/idds.atlas.notifier.rst create mode 100644 docs/source/codes/atlas/idds.atlas.processing.rst create mode 100644 docs/source/codes/atlas/idds.atlas.rst create mode 100644 docs/source/codes/atlas/idds.atlas.rucio.rst create mode 100644 docs/source/codes/atlas/idds.atlas.transformer.rst create mode 100644 docs/source/codes/atlas/idds.atlas.workflow.rst create mode 100644 docs/source/codes/atlas/idds.atlas.workflowv2.rst create mode 100644 docs/source/codes/atlas/idds.rst create mode 100644 docs/source/codes/atlas/modules.rst create mode 100644 docs/source/codes/client/idds.client.rst create mode 100644 docs/source/codes/client/idds.rst create mode 100644 docs/source/codes/client/modules.rst create mode 100644 docs/source/codes/common/idds.common.plugin.rst create mode 100644 docs/source/codes/common/idds.common.rst create mode 100644 docs/source/codes/common/idds.rst create mode 100644 docs/source/codes/common/modules.rst create mode 100644 docs/source/codes/doma/idds.doma.rst create mode 100644 docs/source/codes/doma/idds.doma.workflow.rst create mode 100644 docs/source/codes/doma/idds.doma.workflowv2.rst create mode 100644 docs/source/codes/doma/idds.rst create mode 100644 docs/source/codes/doma/modules.rst create mode 100644 docs/source/codes/libraries.rst create mode 100644 docs/source/codes/main/idds.agents.carrier.rst create mode 100644 docs/source/codes/main/idds.agents.clerk.rst create mode 100644 docs/source/codes/main/idds.agents.common.rst create mode 100644 docs/source/codes/main/idds.agents.conductor.rst create mode 100644 docs/source/codes/main/idds.agents.marshaller.rst create mode 100644 docs/source/codes/main/idds.agents.rst create mode 100644 docs/source/codes/main/idds.agents.transformer.rst create mode 100644 docs/source/codes/main/idds.agents.transporter.rst create mode 100644 docs/source/codes/main/idds.api.rst create mode 100644 docs/source/codes/main/idds.core.rst create mode 100644 docs/source/codes/main/idds.orm.base.rst create mode 100644 docs/source/codes/main/idds.orm.rst create mode 100644 docs/source/codes/main/idds.rest.rst create mode 100644 docs/source/codes/main/idds.rest.v1.rst create mode 100644 docs/source/codes/main/idds.rst create mode 100644 docs/source/codes/main/idds.tests.rst create mode 100644 docs/source/codes/main/modules.rst create mode 100644 docs/source/codes/workflow/idds.rst create mode 100644 docs/source/codes/workflow/idds.workflow.rst create mode 100644 docs/source/codes/workflow/idds.workflowv2.rst create mode 100644 docs/source/codes/workflow/modules.rst diff --git a/docs/source/codes/atlas/idds.atlas.notifier.rst b/docs/source/codes/atlas/idds.atlas.notifier.rst new file mode 100644 index 00000000..278b1da1 --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.notifier.rst @@ -0,0 +1,22 @@ +idds.atlas.notifier package +=========================== + +Submodules +---------- + +idds.atlas.notifier.messaging module +------------------------------------ + +.. automodule:: idds.atlas.notifier.messaging + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas.notifier + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.processing.rst b/docs/source/codes/atlas/idds.atlas.processing.rst new file mode 100644 index 00000000..94b2e267 --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.processing.rst @@ -0,0 +1,102 @@ +idds.atlas.processing package +============================= + +Submodules +---------- + +idds.atlas.processing.activelearning\_condor\_poller module +----------------------------------------------------------- + +.. automodule:: idds.atlas.processing.activelearning_condor_poller + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.activelearning\_condor\_submitter module +-------------------------------------------------------------- + +.. automodule:: idds.atlas.processing.activelearning_condor_submitter + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.base\_plugin module +----------------------------------------- + +.. automodule:: idds.atlas.processing.base_plugin + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.condor\_poller module +------------------------------------------- + +.. automodule:: idds.atlas.processing.condor_poller + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.condor\_submitter module +---------------------------------------------- + +.. automodule:: idds.atlas.processing.condor_submitter + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.hyperparameteropt\_bayesian module +-------------------------------------------------------- + +.. automodule:: idds.atlas.processing.hyperparameteropt_bayesian + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.hyperparameteropt\_condor\_poller module +-------------------------------------------------------------- + +.. automodule:: idds.atlas.processing.hyperparameteropt_condor_poller + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.hyperparameteropt\_condor\_submitter module +----------------------------------------------------------------- + +.. automodule:: idds.atlas.processing.hyperparameteropt_condor_submitter + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.hyperparameteropt\_nevergrad module +--------------------------------------------------------- + +.. automodule:: idds.atlas.processing.hyperparameteropt_nevergrad + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.stagein\_poller module +-------------------------------------------- + +.. automodule:: idds.atlas.processing.stagein_poller + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.processing.stagein\_submitter module +----------------------------------------------- + +.. automodule:: idds.atlas.processing.stagein_submitter + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas.processing + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.rst b/docs/source/codes/atlas/idds.atlas.rst new file mode 100644 index 00000000..716b2f5e --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.rst @@ -0,0 +1,35 @@ +idds.atlas package +================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.atlas.notifier + idds.atlas.processing + idds.atlas.rucio + idds.atlas.transformer + idds.atlas.workflow + idds.atlas.workflowv2 + +Submodules +---------- + +idds.atlas.version module +------------------------- + +.. automodule:: idds.atlas.version + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.rucio.rst b/docs/source/codes/atlas/idds.atlas.rucio.rst new file mode 100644 index 00000000..abb75cae --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.rucio.rst @@ -0,0 +1,78 @@ +idds.atlas.rucio package +======================== + +Submodules +---------- + +idds.atlas.rucio.base\_plugin module +------------------------------------ + +.. automodule:: idds.atlas.rucio.base_plugin + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.collection\_lister module +------------------------------------------ + +.. automodule:: idds.atlas.rucio.collection_lister + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.collection\_metadata\_reader module +---------------------------------------------------- + +.. automodule:: idds.atlas.rucio.collection_metadata_reader + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.contents\_lister module +---------------------------------------- + +.. automodule:: idds.atlas.rucio.contents_lister + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.contents\_register module +------------------------------------------ + +.. automodule:: idds.atlas.rucio.contents_register + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.rule\_creator module +------------------------------------- + +.. automodule:: idds.atlas.rucio.rule_creator + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.rule\_poller module +------------------------------------ + +.. automodule:: idds.atlas.rucio.rule_poller + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.rucio.rule\_submitter module +--------------------------------------- + +.. automodule:: idds.atlas.rucio.rule_submitter + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas.rucio + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.transformer.rst b/docs/source/codes/atlas/idds.atlas.transformer.rst new file mode 100644 index 00000000..3679e6c7 --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.transformer.rst @@ -0,0 +1,46 @@ +idds.atlas.transformer package +============================== + +Submodules +---------- + +idds.atlas.transformer.activelearning\_transformer module +--------------------------------------------------------- + +.. automodule:: idds.atlas.transformer.activelearning_transformer + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.transformer.base\_plugin module +------------------------------------------ + +.. automodule:: idds.atlas.transformer.base_plugin + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.transformer.hyperparameteropt\_transformer module +------------------------------------------------------------ + +.. automodule:: idds.atlas.transformer.hyperparameteropt_transformer + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.transformer.stagein\_transformer module +-------------------------------------------------- + +.. automodule:: idds.atlas.transformer.stagein_transformer + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas.transformer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.workflow.rst b/docs/source/codes/atlas/idds.atlas.workflow.rst new file mode 100644 index 00000000..fe69ed62 --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.workflow.rst @@ -0,0 +1,62 @@ +idds.atlas.workflow package +=========================== + +Submodules +---------- + +idds.atlas.workflow.atlasactuatorwork module +-------------------------------------------- + +.. automodule:: idds.atlas.workflow.atlasactuatorwork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflow.atlascondorwork module +------------------------------------------ + +.. automodule:: idds.atlas.workflow.atlascondorwork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflow.atlasdagwork module +--------------------------------------- + +.. automodule:: idds.atlas.workflow.atlasdagwork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflow.atlashpowork module +--------------------------------------- + +.. automodule:: idds.atlas.workflow.atlashpowork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflow.atlaspandawork module +----------------------------------------- + +.. automodule:: idds.atlas.workflow.atlaspandawork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflow.atlasstageinwork module +------------------------------------------- + +.. automodule:: idds.atlas.workflow.atlasstageinwork + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas.workflow + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.atlas.workflowv2.rst b/docs/source/codes/atlas/idds.atlas.workflowv2.rst new file mode 100644 index 00000000..3c582c80 --- /dev/null +++ b/docs/source/codes/atlas/idds.atlas.workflowv2.rst @@ -0,0 +1,62 @@ +idds.atlas.workflowv2 package +============================= + +Submodules +---------- + +idds.atlas.workflowv2.atlasactuatorwork module +---------------------------------------------- + +.. automodule:: idds.atlas.workflowv2.atlasactuatorwork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflowv2.atlascondorwork module +-------------------------------------------- + +.. automodule:: idds.atlas.workflowv2.atlascondorwork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflowv2.atlasdagwork module +----------------------------------------- + +.. automodule:: idds.atlas.workflowv2.atlasdagwork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflowv2.atlashpowork module +----------------------------------------- + +.. automodule:: idds.atlas.workflowv2.atlashpowork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflowv2.atlaspandawork module +------------------------------------------- + +.. automodule:: idds.atlas.workflowv2.atlaspandawork + :members: + :undoc-members: + :show-inheritance: + +idds.atlas.workflowv2.atlasstageinwork module +--------------------------------------------- + +.. automodule:: idds.atlas.workflowv2.atlasstageinwork + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.atlas.workflowv2 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/idds.rst b/docs/source/codes/atlas/idds.rst new file mode 100644 index 00000000..d942f08f --- /dev/null +++ b/docs/source/codes/atlas/idds.rst @@ -0,0 +1,18 @@ +idds package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.atlas + +Module contents +--------------- + +.. automodule:: idds + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/atlas/modules.rst b/docs/source/codes/atlas/modules.rst new file mode 100644 index 00000000..bc985f17 --- /dev/null +++ b/docs/source/codes/atlas/modules.rst @@ -0,0 +1,7 @@ +idds +==== + +.. toctree:: + :maxdepth: 4 + + idds diff --git a/docs/source/codes/client/idds.client.rst b/docs/source/codes/client/idds.client.rst new file mode 100644 index 00000000..30f3e9b4 --- /dev/null +++ b/docs/source/codes/client/idds.client.rst @@ -0,0 +1,94 @@ +idds.client package +=================== + +Submodules +---------- + +idds.client.base module +----------------------- + +.. automodule:: idds.client.base + :members: + :undoc-members: + :show-inheritance: + +idds.client.cacherclient module +------------------------------- + +.. automodule:: idds.client.cacherclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.catalogclient module +-------------------------------- + +.. automodule:: idds.client.catalogclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.client module +------------------------- + +.. automodule:: idds.client.client + :members: + :undoc-members: + :show-inheritance: + +idds.client.clientmanager module +-------------------------------- + +.. automodule:: idds.client.clientmanager + :members: + :undoc-members: + :show-inheritance: + +idds.client.hpoclient module +---------------------------- + +.. automodule:: idds.client.hpoclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.logsclient module +----------------------------- + +.. automodule:: idds.client.logsclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.messageclient module +-------------------------------- + +.. automodule:: idds.client.messageclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.requestclient module +-------------------------------- + +.. automodule:: idds.client.requestclient + :members: + :undoc-members: + :show-inheritance: + +idds.client.version module +-------------------------- + +.. automodule:: idds.client.version + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.client + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/client/idds.rst b/docs/source/codes/client/idds.rst new file mode 100644 index 00000000..872d8e5c --- /dev/null +++ b/docs/source/codes/client/idds.rst @@ -0,0 +1,18 @@ +idds package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.client + +Module contents +--------------- + +.. automodule:: idds + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/client/modules.rst b/docs/source/codes/client/modules.rst new file mode 100644 index 00000000..bc985f17 --- /dev/null +++ b/docs/source/codes/client/modules.rst @@ -0,0 +1,7 @@ +idds +==== + +.. toctree:: + :maxdepth: 4 + + idds diff --git a/docs/source/codes/common/idds.common.plugin.rst b/docs/source/codes/common/idds.common.plugin.rst new file mode 100644 index 00000000..9c9b99a7 --- /dev/null +++ b/docs/source/codes/common/idds.common.plugin.rst @@ -0,0 +1,30 @@ +idds.common.plugin package +========================== + +Submodules +---------- + +idds.common.plugin.plugin\_base module +-------------------------------------- + +.. automodule:: idds.common.plugin.plugin_base + :members: + :undoc-members: + :show-inheritance: + +idds.common.plugin.plugin\_utils module +--------------------------------------- + +.. automodule:: idds.common.plugin.plugin_utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.common.plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/common/idds.common.rst b/docs/source/codes/common/idds.common.rst new file mode 100644 index 00000000..f04f58cf --- /dev/null +++ b/docs/source/codes/common/idds.common.rst @@ -0,0 +1,78 @@ +idds.common package +=================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.common.plugin + +Submodules +---------- + +idds.common.config module +------------------------- + +.. automodule:: idds.common.config + :members: + :undoc-members: + :show-inheritance: + +idds.common.constants module +---------------------------- + +.. automodule:: idds.common.constants + :members: + :undoc-members: + :show-inheritance: + +idds.common.dict\_class module +------------------------------ + +.. automodule:: idds.common.dict_class + :members: + :undoc-members: + :show-inheritance: + +idds.common.exceptions module +----------------------------- + +.. automodule:: idds.common.exceptions + :members: + :undoc-members: + :show-inheritance: + +idds.common.status\_utils module +-------------------------------- + +.. automodule:: idds.common.status_utils + :members: + :undoc-members: + :show-inheritance: + +idds.common.utils module +------------------------ + +.. automodule:: idds.common.utils + :members: + :undoc-members: + :show-inheritance: + +idds.common.version module +-------------------------- + +.. automodule:: idds.common.version + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.common + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/common/idds.rst b/docs/source/codes/common/idds.rst new file mode 100644 index 00000000..62094b2d --- /dev/null +++ b/docs/source/codes/common/idds.rst @@ -0,0 +1,18 @@ +idds package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.common + +Module contents +--------------- + +.. automodule:: idds + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/common/modules.rst b/docs/source/codes/common/modules.rst new file mode 100644 index 00000000..bc985f17 --- /dev/null +++ b/docs/source/codes/common/modules.rst @@ -0,0 +1,7 @@ +idds +==== + +.. toctree:: + :maxdepth: 4 + + idds diff --git a/docs/source/codes/doma/idds.doma.rst b/docs/source/codes/doma/idds.doma.rst new file mode 100644 index 00000000..6fd50a52 --- /dev/null +++ b/docs/source/codes/doma/idds.doma.rst @@ -0,0 +1,31 @@ +idds.doma package +================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.doma.workflow + idds.doma.workflowv2 + +Submodules +---------- + +idds.doma.version module +------------------------ + +.. automodule:: idds.doma.version + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.doma + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/doma/idds.doma.workflow.rst b/docs/source/codes/doma/idds.doma.workflow.rst new file mode 100644 index 00000000..fb268887 --- /dev/null +++ b/docs/source/codes/doma/idds.doma.workflow.rst @@ -0,0 +1,22 @@ +idds.doma.workflow package +========================== + +Submodules +---------- + +idds.doma.workflow.domapandawork module +--------------------------------------- + +.. automodule:: idds.doma.workflow.domapandawork + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.doma.workflow + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/doma/idds.doma.workflowv2.rst b/docs/source/codes/doma/idds.doma.workflowv2.rst new file mode 100644 index 00000000..29ecde30 --- /dev/null +++ b/docs/source/codes/doma/idds.doma.workflowv2.rst @@ -0,0 +1,22 @@ +idds.doma.workflowv2 package +============================ + +Submodules +---------- + +idds.doma.workflowv2.domapandawork module +----------------------------------------- + +.. automodule:: idds.doma.workflowv2.domapandawork + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.doma.workflowv2 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/doma/idds.rst b/docs/source/codes/doma/idds.rst new file mode 100644 index 00000000..393a52e9 --- /dev/null +++ b/docs/source/codes/doma/idds.rst @@ -0,0 +1,18 @@ +idds package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.doma + +Module contents +--------------- + +.. automodule:: idds + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/doma/modules.rst b/docs/source/codes/doma/modules.rst new file mode 100644 index 00000000..bc985f17 --- /dev/null +++ b/docs/source/codes/doma/modules.rst @@ -0,0 +1,7 @@ +idds +==== + +.. toctree:: + :maxdepth: 4 + + idds diff --git a/docs/source/codes/libraries.rst b/docs/source/codes/libraries.rst new file mode 100644 index 00000000..a01acca5 --- /dev/null +++ b/docs/source/codes/libraries.rst @@ -0,0 +1,64 @@ +iDDS Codes +========== + +iDDS codes documents are automatically generated by sphinx. + +Common libraries +**************** + +Common libaries are the basic libraries which are used by all other libraries. + +.. toctree:: + :maxdepth: 1 + + common/modules + +Workflow libraries +********************* + +Workflow libaries are the basic libraries which are used by all other libraries. + +.. toctree:: + :maxdepth: 1 + + workflow/modules + +Main libraries +**************** + +Main libaries include the core functions, RESTful service and daemon agents which run on the servers. + +.. toctree:: + :maxdepth: 1 + + main/modules + +Client libraries +**************** + +Client libraries is used for users to communicate with iDDS service. + +.. toctree:: + :maxdepth: 1 + + client/modules + +ATLAS libraries +**************** + +ATLAS libraries include plugins for ATLAS special services. + +.. toctree:: + :maxdepth: 1 + + atlas/modules + +DOMA libraries +**************** + +DOMA libraries include plugins for DOMA special services. + +.. toctree:: + :maxdepth: 1 + + doma/modules diff --git a/docs/source/codes/main/idds.agents.carrier.rst b/docs/source/codes/main/idds.agents.carrier.rst new file mode 100644 index 00000000..37a59e7e --- /dev/null +++ b/docs/source/codes/main/idds.agents.carrier.rst @@ -0,0 +1,22 @@ +idds.agents.carrier package +=========================== + +Submodules +---------- + +idds.agents.carrier.carrier module +---------------------------------- + +.. automodule:: idds.agents.carrier.carrier + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.carrier + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.clerk.rst b/docs/source/codes/main/idds.agents.clerk.rst new file mode 100644 index 00000000..87bbec1e --- /dev/null +++ b/docs/source/codes/main/idds.agents.clerk.rst @@ -0,0 +1,22 @@ +idds.agents.clerk package +========================= + +Submodules +---------- + +idds.agents.clerk.clerk module +------------------------------ + +.. automodule:: idds.agents.clerk.clerk + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.clerk + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.common.rst b/docs/source/codes/main/idds.agents.common.rst new file mode 100644 index 00000000..8d9114e9 --- /dev/null +++ b/docs/source/codes/main/idds.agents.common.rst @@ -0,0 +1,38 @@ +idds.agents.common package +========================== + +Submodules +---------- + +idds.agents.common.baseagent module +----------------------------------- + +.. automodule:: idds.agents.common.baseagent + :members: + :undoc-members: + :show-inheritance: + +idds.agents.common.timerscheduler module +---------------------------------------- + +.. automodule:: idds.agents.common.timerscheduler + :members: + :undoc-members: + :show-inheritance: + +idds.agents.common.timertask module +----------------------------------- + +.. automodule:: idds.agents.common.timertask + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.common + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.conductor.rst b/docs/source/codes/main/idds.agents.conductor.rst new file mode 100644 index 00000000..cea25f4b --- /dev/null +++ b/docs/source/codes/main/idds.agents.conductor.rst @@ -0,0 +1,30 @@ +idds.agents.conductor package +============================= + +Submodules +---------- + +idds.agents.conductor.conductor module +-------------------------------------- + +.. automodule:: idds.agents.conductor.conductor + :members: + :undoc-members: + :show-inheritance: + +idds.agents.conductor.consumer module +------------------------------------- + +.. automodule:: idds.agents.conductor.consumer + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.conductor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.marshaller.rst b/docs/source/codes/main/idds.agents.marshaller.rst new file mode 100644 index 00000000..11330bec --- /dev/null +++ b/docs/source/codes/main/idds.agents.marshaller.rst @@ -0,0 +1,22 @@ +idds.agents.marshaller package +============================== + +Submodules +---------- + +idds.agents.marshaller.marshaller module +---------------------------------------- + +.. automodule:: idds.agents.marshaller.marshaller + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.marshaller + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.rst b/docs/source/codes/main/idds.agents.rst new file mode 100644 index 00000000..4935c60c --- /dev/null +++ b/docs/source/codes/main/idds.agents.rst @@ -0,0 +1,36 @@ +idds.agents package +=================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.agents.carrier + idds.agents.clerk + idds.agents.common + idds.agents.conductor + idds.agents.marshaller + idds.agents.transformer + idds.agents.transporter + +Submodules +---------- + +idds.agents.main module +----------------------- + +.. automodule:: idds.agents.main + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.transformer.rst b/docs/source/codes/main/idds.agents.transformer.rst new file mode 100644 index 00000000..cc5746a5 --- /dev/null +++ b/docs/source/codes/main/idds.agents.transformer.rst @@ -0,0 +1,30 @@ +idds.agents.transformer package +=============================== + +Submodules +---------- + +idds.agents.transformer.helper module +------------------------------------- + +.. automodule:: idds.agents.transformer.helper + :members: + :undoc-members: + :show-inheritance: + +idds.agents.transformer.transformer module +------------------------------------------ + +.. automodule:: idds.agents.transformer.transformer + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.transformer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.agents.transporter.rst b/docs/source/codes/main/idds.agents.transporter.rst new file mode 100644 index 00000000..5f4e9653 --- /dev/null +++ b/docs/source/codes/main/idds.agents.transporter.rst @@ -0,0 +1,22 @@ +idds.agents.transporter package +=============================== + +Submodules +---------- + +idds.agents.transporter.transporter module +------------------------------------------ + +.. automodule:: idds.agents.transporter.transporter + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.agents.transporter + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.api.rst b/docs/source/codes/main/idds.api.rst new file mode 100644 index 00000000..f7c99753 --- /dev/null +++ b/docs/source/codes/main/idds.api.rst @@ -0,0 +1,62 @@ +idds.api package +================ + +Submodules +---------- + +idds.api.catalog module +----------------------- + +.. automodule:: idds.api.catalog + :members: + :undoc-members: + :show-inheritance: + +idds.api.collections module +--------------------------- + +.. automodule:: idds.api.collections + :members: + :undoc-members: + :show-inheritance: + +idds.api.contents module +------------------------ + +.. automodule:: idds.api.contents + :members: + :undoc-members: + :show-inheritance: + +idds.api.processings module +--------------------------- + +.. automodule:: idds.api.processings + :members: + :undoc-members: + :show-inheritance: + +idds.api.requests module +------------------------ + +.. automodule:: idds.api.requests + :members: + :undoc-members: + :show-inheritance: + +idds.api.transforms module +-------------------------- + +.. automodule:: idds.api.transforms + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.api + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.core.rst b/docs/source/codes/main/idds.core.rst new file mode 100644 index 00000000..52b144f0 --- /dev/null +++ b/docs/source/codes/main/idds.core.rst @@ -0,0 +1,70 @@ +idds.core package +================= + +Submodules +---------- + +idds.core.catalog module +------------------------ + +.. automodule:: idds.core.catalog + :members: + :undoc-members: + :show-inheritance: + +idds.core.health module +----------------------- + +.. automodule:: idds.core.health + :members: + :undoc-members: + :show-inheritance: + +idds.core.messages module +------------------------- + +.. automodule:: idds.core.messages + :members: + :undoc-members: + :show-inheritance: + +idds.core.processings module +---------------------------- + +.. automodule:: idds.core.processings + :members: + :undoc-members: + :show-inheritance: + +idds.core.requests module +------------------------- + +.. automodule:: idds.core.requests + :members: + :undoc-members: + :show-inheritance: + +idds.core.transforms module +--------------------------- + +.. automodule:: idds.core.transforms + :members: + :undoc-members: + :show-inheritance: + +idds.core.workprogress module +----------------------------- + +.. automodule:: idds.core.workprogress + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.core + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.orm.base.rst b/docs/source/codes/main/idds.orm.base.rst new file mode 100644 index 00000000..8bbc9c10 --- /dev/null +++ b/docs/source/codes/main/idds.orm.base.rst @@ -0,0 +1,54 @@ +idds.orm.base package +===================== + +Submodules +---------- + +idds.orm.base.enum module +------------------------- + +.. automodule:: idds.orm.base.enum + :members: + :undoc-members: + :show-inheritance: + +idds.orm.base.models module +--------------------------- + +.. automodule:: idds.orm.base.models + :members: + :undoc-members: + :show-inheritance: + +idds.orm.base.session module +---------------------------- + +.. automodule:: idds.orm.base.session + :members: + :undoc-members: + :show-inheritance: + +idds.orm.base.types module +-------------------------- + +.. automodule:: idds.orm.base.types + :members: + :undoc-members: + :show-inheritance: + +idds.orm.base.utils module +-------------------------- + +.. automodule:: idds.orm.base.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.orm.base + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.orm.rst b/docs/source/codes/main/idds.orm.rst new file mode 100644 index 00000000..7c91f641 --- /dev/null +++ b/docs/source/codes/main/idds.orm.rst @@ -0,0 +1,86 @@ +idds.orm package +================ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.orm.base + +Submodules +---------- + +idds.orm.collections module +--------------------------- + +.. automodule:: idds.orm.collections + :members: + :undoc-members: + :show-inheritance: + +idds.orm.contents module +------------------------ + +.. automodule:: idds.orm.contents + :members: + :undoc-members: + :show-inheritance: + +idds.orm.health module +---------------------- + +.. automodule:: idds.orm.health + :members: + :undoc-members: + :show-inheritance: + +idds.orm.messages module +------------------------ + +.. automodule:: idds.orm.messages + :members: + :undoc-members: + :show-inheritance: + +idds.orm.processings module +--------------------------- + +.. automodule:: idds.orm.processings + :members: + :undoc-members: + :show-inheritance: + +idds.orm.requests module +------------------------ + +.. automodule:: idds.orm.requests + :members: + :undoc-members: + :show-inheritance: + +idds.orm.transforms module +-------------------------- + +.. automodule:: idds.orm.transforms + :members: + :undoc-members: + :show-inheritance: + +idds.orm.workprogress module +---------------------------- + +.. automodule:: idds.orm.workprogress + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.orm + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.rest.rst b/docs/source/codes/main/idds.rest.rst new file mode 100644 index 00000000..72a7f882 --- /dev/null +++ b/docs/source/codes/main/idds.rest.rst @@ -0,0 +1,18 @@ +idds.rest package +================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.rest.v1 + +Module contents +--------------- + +.. automodule:: idds.rest + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.rest.v1.rst b/docs/source/codes/main/idds.rest.v1.rst new file mode 100644 index 00000000..251a4d88 --- /dev/null +++ b/docs/source/codes/main/idds.rest.v1.rst @@ -0,0 +1,94 @@ +idds.rest.v1 package +==================== + +Submodules +---------- + +idds.rest.v1.app module +----------------------- + +.. automodule:: idds.rest.v1.app + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.cacher module +-------------------------- + +.. automodule:: idds.rest.v1.cacher + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.catalog module +--------------------------- + +.. automodule:: idds.rest.v1.catalog + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.controller module +------------------------------ + +.. automodule:: idds.rest.v1.controller + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.hyperparameteropt module +------------------------------------- + +.. automodule:: idds.rest.v1.hyperparameteropt + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.logs module +------------------------ + +.. automodule:: idds.rest.v1.logs + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.messages module +---------------------------- + +.. automodule:: idds.rest.v1.messages + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.monitor module +--------------------------- + +.. automodule:: idds.rest.v1.monitor + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.requests module +---------------------------- + +.. automodule:: idds.rest.v1.requests + :members: + :undoc-members: + :show-inheritance: + +idds.rest.v1.utils module +------------------------- + +.. automodule:: idds.rest.v1.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.rest.v1 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.rst b/docs/source/codes/main/idds.rst new file mode 100644 index 00000000..ed271e11 --- /dev/null +++ b/docs/source/codes/main/idds.rst @@ -0,0 +1,35 @@ +idds package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.agents + idds.api + idds.core + idds.orm + idds.rest + idds.tests + +Submodules +---------- + +idds.version module +------------------- + +.. automodule:: idds.version + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/idds.tests.rst b/docs/source/codes/main/idds.tests.rst new file mode 100644 index 00000000..0bb78901 --- /dev/null +++ b/docs/source/codes/main/idds.tests.rst @@ -0,0 +1,350 @@ +idds.tests package +================== + +Submodules +---------- + +idds.tests.activelearning\_test module +-------------------------------------- + +.. automodule:: idds.tests.activelearning_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.cacher\_test module +------------------------------ + +.. automodule:: idds.tests.cacher_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.catalog\_test module +------------------------------- + +.. automodule:: idds.tests.catalog_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.client\_test module +------------------------------ + +.. automodule:: idds.tests.client_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.common module +------------------------ + +.. automodule:: idds.tests.common + :members: + :undoc-members: + :show-inheritance: + +idds.tests.core\_tests module +----------------------------- + +.. automodule:: idds.tests.core_tests + :members: + :undoc-members: + :show-inheritance: + +idds.tests.datacarousel\_test module +------------------------------------ + +.. automodule:: idds.tests.datacarousel_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.hyperparameteropt\_bayesian\_test module +--------------------------------------------------- + +.. automodule:: idds.tests.hyperparameteropt_bayesian_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.hyperparameteropt\_client\_test module +------------------------------------------------- + +.. automodule:: idds.tests.hyperparameteropt_client_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.hyperparameteropt\_docker\_local\_test module +-------------------------------------------------------- + +.. automodule:: idds.tests.hyperparameteropt_docker_local_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.hyperparameteropt\_docker\_test module +------------------------------------------------- + +.. automodule:: idds.tests.hyperparameteropt_docker_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.hyperparameteropt\_nevergrad\_test module +---------------------------------------------------- + +.. automodule:: idds.tests.hyperparameteropt_nevergrad_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.logs\_test module +---------------------------- + +.. automodule:: idds.tests.logs_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.message\_test module +------------------------------- + +.. automodule:: idds.tests.message_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.message\_test1 module +-------------------------------- + +.. automodule:: idds.tests.message_test1 + :members: + :undoc-members: + :show-inheritance: + +idds.tests.migrating\_requests\_v1\_to\_v2 module +------------------------------------------------- + +.. automodule:: idds.tests.migrating_requests_v1_to_v2 + :members: + :undoc-members: + :show-inheritance: + +idds.tests.panda\_iam\_test module +---------------------------------- + +.. automodule:: idds.tests.panda_iam_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.panda\_test module +----------------------------- + +.. automodule:: idds.tests.panda_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.performance\_test\_with\_cx\_oracle module +----------------------------------------------------- + +.. automodule:: idds.tests.performance_test_with_cx_oracle + :members: + :undoc-members: + :show-inheritance: + +idds.tests.performance\_test\_with\_sqlalchemy module +----------------------------------------------------- + +.. automodule:: idds.tests.performance_test_with_sqlalchemy + :members: + :undoc-members: + :show-inheritance: + +idds.tests.rest\_test module +---------------------------- + +.. automodule:: idds.tests.rest_test + :members: + :undoc-members: + :show-inheritance: + +idds.tests.run\_sql module +-------------------------- + +.. automodule:: idds.tests.run_sql + :members: + :undoc-members: + :show-inheritance: + +idds.tests.scaling\_checks module +--------------------------------- + +.. automodule:: idds.tests.scaling_checks + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_activelearning module +-------------------------------------- + +.. automodule:: idds.tests.test_activelearning + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_atlaspandawork module +-------------------------------------- + +.. automodule:: idds.tests.test_atlaspandawork + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_catalog module +------------------------------- + +.. automodule:: idds.tests.test_catalog + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_datacarousel module +------------------------------------ + +.. automodule:: idds.tests.test_datacarousel + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_domapanda module +--------------------------------- + +.. automodule:: idds.tests.test_domapanda + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_domapanda\_workflow module +------------------------------------------- + +.. automodule:: idds.tests.test_domapanda_workflow + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_hyperparameteropt module +----------------------------------------- + +.. automodule:: idds.tests.test_hyperparameteropt + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_migrate\_requests module +----------------------------------------- + +.. automodule:: idds.tests.test_migrate_requests + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_property module +-------------------------------- + +.. automodule:: idds.tests.test_property + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_request\_transform module +------------------------------------------ + +.. automodule:: idds.tests.test_request_transform + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_requests module +-------------------------------- + +.. automodule:: idds.tests.test_requests + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_running\_data module +------------------------------------- + +.. automodule:: idds.tests.test_running_data + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_scaling module +------------------------------- + +.. automodule:: idds.tests.test_scaling + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_transform\_collection\_content module +------------------------------------------------------ + +.. automodule:: idds.tests.test_transform_collection_content + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_transform\_processing module +--------------------------------------------- + +.. automodule:: idds.tests.test_transform_processing + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_workflow module +-------------------------------- + +.. automodule:: idds.tests.test_workflow + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_workflow\_condition module +------------------------------------------- + +.. automodule:: idds.tests.test_workflow_condition + :members: + :undoc-members: + :show-inheritance: + +idds.tests.test\_workflow\_condition\_v2 module +----------------------------------------------- + +.. automodule:: idds.tests.test_workflow_condition_v2 + :members: + :undoc-members: + :show-inheritance: + +idds.tests.trigger\_release module +---------------------------------- + +.. automodule:: idds.tests.trigger_release + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/main/modules.rst b/docs/source/codes/main/modules.rst new file mode 100644 index 00000000..bc985f17 --- /dev/null +++ b/docs/source/codes/main/modules.rst @@ -0,0 +1,7 @@ +idds +==== + +.. toctree:: + :maxdepth: 4 + + idds diff --git a/docs/source/codes/workflow/idds.rst b/docs/source/codes/workflow/idds.rst new file mode 100644 index 00000000..6e87fe9c --- /dev/null +++ b/docs/source/codes/workflow/idds.rst @@ -0,0 +1,19 @@ +idds package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + idds.workflow + idds.workflowv2 + +Module contents +--------------- + +.. automodule:: idds + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/workflow/idds.workflow.rst b/docs/source/codes/workflow/idds.workflow.rst new file mode 100644 index 00000000..28a48529 --- /dev/null +++ b/docs/source/codes/workflow/idds.workflow.rst @@ -0,0 +1,62 @@ +idds.workflow package +===================== + +Submodules +---------- + +idds.workflow.base module +------------------------- + +.. automodule:: idds.workflow.base + :members: + :undoc-members: + :show-inheritance: + +idds.workflow.version module +---------------------------- + +.. automodule:: idds.workflow.version + :members: + :undoc-members: + :show-inheritance: + +idds.workflow.work module +------------------------- + +.. automodule:: idds.workflow.work + :members: + :undoc-members: + :show-inheritance: + +idds.workflow.workflow module +----------------------------- + +.. automodule:: idds.workflow.workflow + :members: + :undoc-members: + :show-inheritance: + +idds.workflow.workflow1 module +------------------------------ + +.. automodule:: idds.workflow.workflow1 + :members: + :undoc-members: + :show-inheritance: + +idds.workflow.workflowv2 module +------------------------------- + +.. automodule:: idds.workflow.workflowv2 + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.workflow + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/workflow/idds.workflowv2.rst b/docs/source/codes/workflow/idds.workflowv2.rst new file mode 100644 index 00000000..b458bd68 --- /dev/null +++ b/docs/source/codes/workflow/idds.workflowv2.rst @@ -0,0 +1,46 @@ +idds.workflowv2 package +======================= + +Submodules +---------- + +idds.workflowv2.base module +--------------------------- + +.. automodule:: idds.workflowv2.base + :members: + :undoc-members: + :show-inheritance: + +idds.workflowv2.version module +------------------------------ + +.. automodule:: idds.workflowv2.version + :members: + :undoc-members: + :show-inheritance: + +idds.workflowv2.work module +--------------------------- + +.. automodule:: idds.workflowv2.work + :members: + :undoc-members: + :show-inheritance: + +idds.workflowv2.workflow module +------------------------------- + +.. automodule:: idds.workflowv2.workflow + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: idds.workflowv2 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/codes/workflow/modules.rst b/docs/source/codes/workflow/modules.rst new file mode 100644 index 00000000..bc985f17 --- /dev/null +++ b/docs/source/codes/workflow/modules.rst @@ -0,0 +1,7 @@ +idds +==== + +.. toctree:: + :maxdepth: 4 + + idds From 86e266654564157a89fa0aae4410b1d6b3824968 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Nov 2021 15:06:06 +0100 Subject: [PATCH 127/156] to update local parameters with global parameters or class attributes --- .../idds/atlas/workflowv2/atlaspandawork.py | 45 +++++++++++++++++++ workflow/lib/idds/workflowv2/work.py | 3 ++ workflow/lib/idds/workflowv2/workflow.py | 15 +++++++ 3 files changed, 63 insertions(+) diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py index a36a4f4f..7598c16c 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -186,6 +186,51 @@ def parse_task_parameters(self, task_parameters): # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) self.add_errors(str(ex)) + def renew_parameter(self, parameter): + new_parameter = parameter + if '__idds__' in parameter: + pos_start = parameter.find('__idds__') + attr = parameter[pos_start:] + attr = attr.replace("__idds__", "") + pos = attr.find("__") + if pos > -1: + attr = attr[:pos] + if attr: + idds_attr = "__idds__" + attr + "__" + if hasattr(self, attr): + attr_value = getattr(self, attr) + new_parameter = parameter.replace(idds_attr, attr_value) + return new_parameter + + def renew_parameters_from_attributes(self): + if not self.task_parameters: + return + + try: + if 'taskName' in self.task_parameters: + self.task_name = self.task_parameters['taskName'] + self.task_name = self.renew_parameter(self.task_name) + self.set_work_name(self.task_name) + + if 'prodSourceLabel' in self.task_parameters: + self.task_type = self.task_parameters['prodSourceLabel'] + + if 'jobParameters' in self.task_parameters: + jobParameters = self.task_parameters['jobParameters'] + for jobP in jobParameters: + if type(jobP) in [dict]: + for key in jobP: + if jobP[key] and type(jobP[key]) in [str]: + jobP[key] = self.renew_parameter(jobP[key]) + for coll_id in self.collections: + coll_name = self.collections[coll_id].name + self.collections[coll_id].name = self.renew_parameter(coll_name) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + self.add_errors(str(ex)) + def get_rucio_client(self): try: client = RucioClient() diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index b97d2568..123f03ec 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -1013,6 +1013,9 @@ def sync_global_parameters(self, global_parameters): for key in global_parameters: setattr(self, key, global_parameters[key]) + def renew_parameters_from_attributes(self): + pass + def setup_logger(self): """ Setup logger diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 398df75a..5bbd753c 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -1067,6 +1067,7 @@ def get_new_work_to_run(self, work_id, new_parameters=None): work.initialize_work() work.sync_global_parameters(self.global_parameters) + work.renew_parameters_from_attributes() work.num_run = self.num_run works = self.works self.works = works @@ -1078,6 +1079,19 @@ def get_new_work_to_run(self, work_id, new_parameters=None): return work + def get_new_parameters_for_work(self, work): + new_parameters = self.get_destination_parameters(work.get_internal_id()) + if new_parameters: + work.set_parameters(new_parameters) + work.sequence_id = self.num_total_works + + work.initialize_work() + work.sync_global_parameters(self.global_parameters) + work.renew_parameters_from_attributes() + works = self.works + self.works = works + return work + def register_user_defined_condition(self, condition): cond_src = inspect.getsource(condition) self.user_defined_conditions[condition.__name__] = cond_src @@ -1283,6 +1297,7 @@ def get_new_works(self): works = [] for k in self.new_to_run_works: if isinstance(self.works[k], Work): + self.works[k] = self.get_new_parameters_for_work(self.works[k]) works.append(self.works[k]) if isinstance(self.works[k], Workflow): works = works + self.works[k].get_new_works() From 707b1486758af6c773eb9f590c0e208a63ad5874 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Nov 2021 15:18:54 +0100 Subject: [PATCH 128/156] fix missing get_primary_input_collection --- workflow/lib/idds/workflowv2/workflow.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 5bbd753c..5fe1befe 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -1343,14 +1343,26 @@ def get_primary_initial_collection(self): """ if self.primary_initial_work: - return self.get_works()[self.primary_initial_work].get_primary_input_collection() + if isinstance(self.get_works()[self.primary_initial_work], Workflow): + return self.get_works()[self.primary_initial_work].get_primary_initial_collection() + else: + return self.get_works()[self.primary_initial_work].get_primary_input_collection() elif self.initial_works: - return self.get_works()[self.initial_works[0]].get_primary_input_collection() + if isinstance(self.get_works()[self.initial_works[0]], Workflow): + return self.get_works()[self.initial_works[0]].get_primary_initial_collection() + else: + return self.get_works()[self.initial_works[0]].get_primary_input_collection() elif self.independent_works: - return self.get_works()[self.independent_works[0]].get_primary_input_collection() + if isinstance(self.get_works()[self.independent_works[0]], Workflow): + return self.get_works()[self.independent_works[0]].get_primary_initial_collection() + else: + return self.get_works()[self.independent_works[0]].get_primary_input_collection() else: keys = self.get_works().keys() - return self.get_works()[keys[0]].get_primary_input_collection() + if isinstance(self.get_works()[keys[0]], Workflow): + return self.get_works()[keys[0]].get_primary_initial_collection() + else: + return self.get_works()[keys[0]].get_primary_input_collection() return None def get_dependency_works(self, work_id, depth, max_depth): From a6cd5cff1def222b760e6b24cbe890c57ad824ef Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Nov 2021 15:35:52 +0100 Subject: [PATCH 129/156] add is_client --- common/lib/idds/common/utils.py | 15 +++++++++++++++ main/etc/idds/idds.cfg.client.template | 13 ------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index c89d873a..a52981e9 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -476,3 +476,18 @@ def extract_scope_atlas(did, scopes): def truncate_string(string, length=800): string = (string[:length] + '...') if string and len(string) > length else string return string + + +def is_client(): + if 'IDDS_CLIENT_MODE' not in os.environ: + if config_has_section('database') and config_has_option('database', 'default'): + client_mode = False + else: + client_mode = True + else: + if os.environ['IDDS_CLIENT_MODE']: + client_mode = True + else: + client_mode = False + + return client_mode diff --git a/main/etc/idds/idds.cfg.client.template b/main/etc/idds/idds.cfg.client.template index e2de5947..421b17b7 100755 --- a/main/etc/idds/idds.cfg.client.template +++ b/main/etc/idds/idds.cfg.client.template @@ -12,19 +12,6 @@ # logdir = /var/log/idds loglevel = DEBUG -[database] -#default = mysql://idds:idds@pcuwvirt5.cern.ch/idds -#default = mysql://idds:idds_passwd@aipanda182.cern.ch/idds -#default = sqlite:////tmp/idds.db -#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________))) -#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________))) - -#default = postgresql://idds:idds@localhost/idds -pool_size=20 -pool_recycle=3600 -echo=0 -pool_reset_on_return=rollback - [rest] host = https://aipanda181.cern.ch:443/idds #url_prefix = /idds From 99520e2ce8e1d095535ac22bbb2e17072a4bb5b1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Nov 2021 15:36:28 +0100 Subject: [PATCH 130/156] new version 0.8.5 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 0ef19cd5..8bdc4529 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 0d779dd8..79348ead 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.4 - - idds-workflow==0.8.4 \ No newline at end of file + - idds-common==0.8.5 + - idds-workflow==0.8.5 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 0ef19cd5..8bdc4529 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 9b818272..029d894d 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.4 - - idds-workflow==0.8.4 \ No newline at end of file + - idds-common==0.8.5 + - idds-workflow==0.8.5 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 0ef19cd5..8bdc4529 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 642205d0..6b2ffb5b 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index d932e757..a0d5c61d 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.4 - - idds-workflow==0.8.4 \ No newline at end of file + - idds-common==0.8.5 + - idds-workflow==0.8.5 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 0ef19cd5..8bdc4529 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index fde1f914..de3a7065 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.4 - - idds-workflow==0.8.4 - - idds-client==0.8.4 \ No newline at end of file + - idds-common==0.8.5 + - idds-workflow==0.8.5 + - idds-client==0.8.5 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 0ef19cd5..8bdc4529 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/website/version.py b/website/version.py index 0ef19cd5..8bdc4529 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 0ef19cd5..8bdc4529 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.4" +release_version = "0.8.5" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index d1e30ba3..6b2a516e 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.4 \ No newline at end of file + - idds-common==0.8.5 \ No newline at end of file From 43e62bc29f6b1b1beda632fb57a2756f91cb9d5f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 15 Nov 2021 20:48:45 +0100 Subject: [PATCH 131/156] update poll input dependency --- .../idds/atlas/workflowv2/atlaspandawork.py | 4 +-- .../idds/agents/transformer/transformer.py | 35 ++++++++++++++++--- main/lib/idds/core/transforms.py | 27 ++++++++++++++ monitor/conf.js | 12 +++---- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py index 7598c16c..777ebbf3 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -752,9 +752,9 @@ def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True self.logger.debug("syn_work_status, self.active_processings: %s" % str(self.active_processings)) self.logger.debug("syn_work_status, self.has_new_inputs(): %s" % str(self.has_new_inputs)) self.logger.debug("syn_work_status, coll_metadata_is_open: %s" % - str(self.collections[self.primary_input_collection].coll_metadata['is_open'])) + str(self.collections[self._primary_input_collection].coll_metadata['is_open'])) self.logger.debug("syn_work_status, primary_input_collection_status: %s" % - str(self.collections[self.primary_input_collection].status)) + str(self.collections[self._primary_input_collection].status)) self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) self.logger.debug("syn_work_status(%s): is_input_collections_closed: %s" % (str(self.get_processing_ids()), str(self.is_input_collections_closed()))) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 98fbffde..619ab04a 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -209,6 +209,12 @@ def is_all_inputs_dependency_terminated(self, inputs_dependency): return False return True + def is_input_dependency_terminated(self, input_dependency): + if input_dependency['status'] in [ContentStatus.Available, ContentStatus.FakeAvailable, + ContentStatus.FinalFailed, ContentStatus.Missing]: + return True + return False + def get_updated_contents(self, transform, registered_input_output_maps): updated_contents = [] updated_input_contents_full, updated_output_contents_full = [], [] @@ -298,6 +304,22 @@ def trigger_release_inputs(self, updated_output_contents, work, input_output_map self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) return updated_contents + def poll_inputs_dependency(self, transform, registered_input_output_maps): + unfinished_inputs = {} + for map_id in registered_input_output_maps: + inputs_dependency = registered_input_output_maps[map_id]['inputs_dependency'] if 'inputs_dependency' in registered_input_output_maps[map_id] else [] + for content in inputs_dependency: + if (content['status'] not in [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing] + and content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing]): # noqa W503 + if content['coll_id'] not in unfinished_inputs: + unfinished_inputs[content['coll_id']] = [] + unfinished_inputs[content['coll_id']].append(content) + + # updated_contents = core_transforms.release_inputs(to_release_inputs) + updated_contents = core_transforms.poll_inputs_dependency_by_collection(unfinished_inputs) + self.logger.debug("poll_inputs_dependency, updated_contents: %s" % str(updated_contents)) + return updated_contents + def process_new_transform_real(self, transform): """ Process new transform @@ -850,8 +872,10 @@ def process_running_transform_real(self, transform): updated_contents, updated_input_contents_full, updated_output_contents_full = self.get_updated_contents(transform, registered_input_output_maps) # if work.use_dependency_to_release_jobs() and (updated_output_contents_full or work.has_to_release_inputs()): if work.use_dependency_to_release_jobs(): - self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) - to_release_input_contents = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps) + pass + # self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) + # to_release_input_contents = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps) + to_release_input_contents = self.poll_inputs_dependency(transform, registered_input_output_maps) self.logger.info("generate_message: %s" % transform['transform_id']) if new_input_contents: @@ -877,9 +901,10 @@ def process_running_transform_real(self, transform): if work.is_terminated(): self.logger.info("Transform(%s) work is terminated, trigger to release all final status files" % (transform['transform_id'])) if work.use_dependency_to_release_jobs(): - self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) - to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps, final=True) - to_release_input_contents = to_release_input_contents + to_release_input_contents1 + pass + # self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) + # to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps, final=True) + # to_release_input_contents = to_release_input_contents + to_release_input_contents1 to_resume_transform = False reactivated_contents = [] diff --git a/main/lib/idds/core/transforms.py b/main/lib/idds/core/transforms.py index ff9e5769..4fd577d6 100644 --- a/main/lib/idds/core/transforms.py +++ b/main/lib/idds/core/transforms.py @@ -516,6 +516,33 @@ def release_inputs_by_collection(to_release_inputs, final=False): return update_contents +def poll_inputs_dependency_by_collection(unfinished_inputs): + update_contents = [] + status_to_check = [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed, ContentStatus.Missing] + for coll_id in unfinished_inputs: + unfinished_contents = unfinished_inputs[coll_id] + contents = orm_contents.get_input_contents(request_id=unfinished_contents[0]['request_id'], + coll_id=unfinished_contents[0]['coll_id'], + name=None) + + to_release_status = {} + for content in contents: + if (content['content_relation_type'] == ContentRelationType.Output): # noqa: W503 + if content['status'] in status_to_check: + to_release_status[content['name']] = content['status'] + elif content['substatus'] in status_to_check: + to_release_status[content['name']] = content['substatus'] + for content in unfinished_contents: + if content['name'] in to_release_status: + if (content['status'] != to_release_status[content['name']]): + update_content = {'content_id': content['content_id'], + 'substatus': to_release_status[content['name']], + 'status': to_release_status[content['name']]} + update_contents.append(update_content) + + return update_contents + + def get_work_name_to_coll_map(request_id): tfs = orm_transforms.get_transforms(request_id=request_id) colls = orm_collections.get_collections(request_id=request_id) diff --git a/monitor/conf.js b/monitor/conf.js index fe429e62..e559ac83 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus781.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus781.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus781.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus781.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus781.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus781.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus739.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus739.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus739.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/false/true" } From f736a8314b1dcc40aafbfc40518a45c07094a4be Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 16 Nov 2021 15:29:52 +0100 Subject: [PATCH 132/156] fix clean_works --- workflow/lib/idds/workflowv2/workflow.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 5fe1befe..4ffd17d7 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -1920,8 +1920,11 @@ def resume_works(self): self.runs[str(self.num_run)].resume_works() def clean_works(self): - if self.runs: - self.runs[str(self.num_run)].clean_works() + # if self.runs: + # self.runs[str(self.num_run)].clean_works() + self.parent_num_run = None + self._num_run = 0 + self.runs = {} def is_to_expire(self, expired_at=None, pending_time=None, request_id=None): if self.runs: From be703fb0281f51866e4dcccb5ac046fbdd35d9cd Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 16 Nov 2021 15:30:52 +0100 Subject: [PATCH 133/156] fix replace parameters --- atlas/lib/idds/atlas/workflowv2/atlaspandawork.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py index 777ebbf3..78682696 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -188,18 +188,18 @@ def parse_task_parameters(self, task_parameters): def renew_parameter(self, parameter): new_parameter = parameter - if '__idds__' in parameter: - pos_start = parameter.find('__idds__') + if '___idds___' in parameter: + pos_start = parameter.find('___idds___') attr = parameter[pos_start:] - attr = attr.replace("__idds__", "") - pos = attr.find("__") + attr = attr.replace("___idds___", "") + pos = attr.find("___") if pos > -1: attr = attr[:pos] if attr: - idds_attr = "__idds__" + attr + "__" + idds_attr = "___idds___" + attr + "___" if hasattr(self, attr): attr_value = getattr(self, attr) - new_parameter = parameter.replace(idds_attr, attr_value) + new_parameter = parameter.replace(idds_attr, str(attr_value)) return new_parameter def renew_parameters_from_attributes(self): From ab87c8ec2db4890bd7ea09d0ef2df0296a601de2 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 16 Nov 2021 17:22:24 +0100 Subject: [PATCH 134/156] propagate request_id to work --- doma/lib/idds/doma/workflow/domapandawork.py | 4 +++- .../lib/idds/doma/workflowv2/domapandawork.py | 4 +++- main/lib/idds/agents/clerk/clerk.py | 5 ++++- main/lib/idds/tests/core_tests.py | 14 ++++++------- main/lib/idds/tests/test_migrate_requests.py | 8 ++++---- workflow/lib/idds/workflow/work.py | 13 ++++++++++++ workflow/lib/idds/workflowv2/work.py | 20 ++++++++++++++++--- 7 files changed, 51 insertions(+), 17 deletions(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index 32d2b321..e93eba67 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -345,7 +345,7 @@ def create_processing(self, input_output_maps=[]): :param input_output_maps: new maps from inputs to outputs. """ # avoid duplicated task name - self.task_name = self.task_name + "_" + str(self.get_work_id()) + self.task_name = self.task_name + "_" + str(self.get_request_id()) + "_" + str(self.get_work_id()) in_files = [] for job in self.dependency_map: @@ -397,6 +397,8 @@ def create_processing(self, input_output_maps=[]): }, ] + task_param_map['reqID'] = self.get_work_id() + processing_metadata = {'task_param': task_param_map} proc = Processing(processing_metadata=processing_metadata) proc.workload_id = None diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index 9cfe7495..9ba00a64 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -350,7 +350,7 @@ def create_processing(self, input_output_maps=[]): :param input_output_maps: new maps from inputs to outputs. """ # avoid duplicated task name - self.task_name = self.task_name + "_" + str(self.get_work_id()) + self.task_name = self.task_name + "_" + str(self.get_request_id()) + "_" + str(self.get_work_id()) in_files = [] for job in self.dependency_map: @@ -402,6 +402,8 @@ def create_processing(self, input_output_maps=[]): }, ] + task_param_map['reqID'] = self.get_work_id() + processing_metadata = {'task_param': task_param_map} proc = Processing(processing_metadata=processing_metadata) proc.workload_id = None diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index c64fb074..31354ed9 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -6,7 +6,7 @@ # http://www.apache.org/licenses/LICENSE-2.0OA # # Authors: -# - Wen Guan, , 2019 +# - Wen Guan, , 2019 - 2021 import datetime import traceback @@ -72,6 +72,8 @@ def show_queue_size(self): def generate_transform(self, req, work): wf = req['request_metadata']['workflow'] + work.set_request_id(req['request_id']) + new_transform = {'request_id': req['request_id'], 'workload_id': req['workload_id'], 'transform_type': work.get_work_type(), @@ -138,6 +140,7 @@ def process_new_request(self, req): # new_work = work.copy() new_work = work new_work.add_proxy(wf.get_proxy()) + # new_work.set_request_id(req['request_id']) # new_work.create_processing() transform = self.generate_transform(req, work) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 22c4acb0..80ad550a 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,15 +102,15 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=211, with_detail=True, with_metadata=True) +reqs = get_requests(request_id=219, with_detail=True, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) - print(json_dumps(req, sort_keys=True, indent=4)) + # print(json_dumps(req, sort_keys=True, indent=4)) # show_works(req) pass -sys.exit(0) +# sys.exit(0) """ # reqs = get_requests() @@ -126,14 +126,14 @@ def show_works(req): """ -tfs = get_transforms(request_id=205) +tfs = get_transforms(request_id=219) for tf in tfs: # print(tf) # print(tf['transform_metadata']['work'].to_dict()) - print(json_dumps(tf, sort_keys=True, indent=4)) + # print(json_dumps(tf, sort_keys=True, indent=4)) pass -sys.exit(0) +# sys.exit(0) """ msgs = retrieve_messages(workload_id=25972557) @@ -154,7 +154,7 @@ def show_works(req): sys.exit(0) """ -prs = get_processings(request_id=205) +prs = get_processings(request_id=219) i = 0 for pr in prs: # if pr['request_id'] == 91: diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index 8814be3d..ee6596d2 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -29,16 +29,16 @@ def migrate(): # atlas atlas_host = 'https://aipanda181.cern.ch:443/idds' # noqa F841 - cm1 = ClientManager(host=atlas_host) + cm1 = ClientManager(host=dev_host) # reqs = cm1.get_requests(request_id=290) - old_request_id = 72521 + old_request_id = 215 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 # for old_request_id in [200]: # noqa E115 - for old_request_id in [72521]: # noqa E115 # doma 183 + for old_request_id in [old_request_id]: # noqa E115 # doma 183 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=atlas_host) + cm2 = ClientManager(host=dev_host) for req in reqs: req = convert_old_req_2_workflow_req(req) workflow = req['request_metadata']['workflow'] diff --git a/workflow/lib/idds/workflow/work.py b/workflow/lib/idds/workflow/work.py index 8051a079..a69545a9 100644 --- a/workflow/lib/idds/workflow/work.py +++ b/workflow/lib/idds/workflow/work.py @@ -872,6 +872,19 @@ def get_work_id(self): """ return self.work_id + def set_request_id(self, request_id): + """ + *** Function called by Marshaller and clerk agent. + *** It's the transform_id set by core_workprogresses + """ + self.request_id = request_id + + def get_request_id(self): + """ + *** Function called by Marshaller and clerk agent. + """ + return self.request_id + # def set_workflow(self, workflow): # self.workflow = workflow diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 123f03ec..4557859f 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -397,7 +397,7 @@ def has_new_updates(self): class Work(Base): def __init__(self, executable=None, arguments=None, parameters=None, setup=None, work_type=None, - work_tag=None, exec_type='local', sandbox=None, work_id=None, work_name=None, + work_tag=None, exec_type='local', sandbox=None, request_id=None, work_id=None, work_name=None, primary_input_collection=None, other_input_collections=None, input_collections=None, primary_output_collection=None, other_output_collections=None, output_collections=None, log_collections=None, release_inputs_after_submitting=False, @@ -449,6 +449,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.work_tag = work_tag self.exec_type = exec_type self.sandbox = sandbox + self.request_id = request_id self.work_id = work_id self.work_name = work_name if not self.work_name: @@ -1030,7 +1031,7 @@ def get_errors(self): def set_work_id(self, work_id, transforming=True): """ - *** Function called by Marshaller agent. + *** Function called by Marshaller and clerk agent. *** It's the transform_id set by core_workprogresses """ self.work_id = work_id @@ -1038,10 +1039,23 @@ def set_work_id(self, work_id, transforming=True): def get_work_id(self): """ - *** Function called by Marshaller agent. + *** Function called by Marshaller and clerk agent. """ return self.work_id + def set_request_id(self, request_id): + """ + *** Function called by Marshaller and clerk agent. + *** It's the transform_id set by core_workprogresses + """ + self.request_id = request_id + + def get_request_id(self): + """ + *** Function called by Marshaller and clerk agent. + """ + return self.request_id + # def set_workflow(self, workflow): # self.workflow = workflow From bc4f3ad75d2b0a8dd9c6b117282a807826eadd35 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Wed, 17 Nov 2021 13:05:29 +0100 Subject: [PATCH 135/156] new version 0.8.6 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/lib/idds/doma/workflow/domapandawork.py | 2 +- doma/lib/idds/doma/workflowv2/domapandawork.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/tests/test_domapanda.py | 1 + main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 16 files changed, 21 insertions(+), 20 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 8bdc4529..3a62034a 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 79348ead..f82788bb 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.5 - - idds-workflow==0.8.5 \ No newline at end of file + - idds-common==0.8.6 + - idds-workflow==0.8.6 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 8bdc4529..3a62034a 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 029d894d..2d3ebf8d 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.5 - - idds-workflow==0.8.5 \ No newline at end of file + - idds-common==0.8.6 + - idds-workflow==0.8.6 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 8bdc4529..3a62034a 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 6b2ffb5b..64b532d2 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index e93eba67..7ab5a23d 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -397,7 +397,7 @@ def create_processing(self, input_output_maps=[]): }, ] - task_param_map['reqID'] = self.get_work_id() + task_param_map['reqID'] = self.get_request_id() processing_metadata = {'task_param': task_param_map} proc = Processing(processing_metadata=processing_metadata) diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index 9ba00a64..e2f46ff4 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -402,7 +402,7 @@ def create_processing(self, input_output_maps=[]): }, ] - task_param_map['reqID'] = self.get_work_id() + task_param_map['reqID'] = self.get_request_id() processing_metadata = {'task_param': task_param_map} proc = Processing(processing_metadata=processing_metadata) diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index a0d5c61d..23885dde 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.5 - - idds-workflow==0.8.5 \ No newline at end of file + - idds-common==0.8.6 + - idds-workflow==0.8.6 \ No newline at end of file diff --git a/main/lib/idds/tests/test_domapanda.py b/main/lib/idds/tests/test_domapanda.py index 70ec5a73..e15b6bbc 100644 --- a/main/lib/idds/tests/test_domapanda.py +++ b/main/lib/idds/tests/test_domapanda.py @@ -37,6 +37,7 @@ task_queue = 'DOMA_LSST_GOOGLE_TEST' +task_queue = 'DOMA_LSST_GOOGLE_MERGE' def randStr(chars=string.ascii_lowercase + string.digits, N=10): diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 8bdc4529..3a62034a 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index de3a7065..3813dcf8 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.5 - - idds-workflow==0.8.5 - - idds-client==0.8.5 \ No newline at end of file + - idds-common==0.8.6 + - idds-workflow==0.8.6 + - idds-client==0.8.6 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 8bdc4529..3a62034a 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/website/version.py b/website/version.py index 8bdc4529..3a62034a 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 8bdc4529..3a62034a 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.5" +release_version = "0.8.6" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 6b2a516e..c1966408 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.5 \ No newline at end of file + - idds-common==0.8.6 \ No newline at end of file From 3261708f8bc5df2860d38312a82bc079ef8737fc Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 19 Nov 2021 15:14:37 +0100 Subject: [PATCH 136/156] optimize trigger release jobs --- .../idds/agents/transformer/transformer.py | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 619ab04a..10b7a882 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -225,7 +225,7 @@ def get_updated_contents(self, transform, registered_input_output_maps): inputs_dependency = registered_input_output_maps[map_id]['inputs_dependency'] if 'inputs_dependency' in registered_input_output_maps[map_id] else [] if self.is_all_inputs_dependency_available(inputs_dependency): - self.logger.debug("all input dependency available: %s, inputs: %s" % (str(inputs_dependency), str(inputs))) + # self.logger.debug("all input dependency available: %s, inputs: %s" % (str(inputs_dependency), str(inputs))) for content in inputs: content['substatus'] = ContentStatus.Available if content['status'] != content['substatus']: @@ -236,7 +236,7 @@ def get_updated_contents(self, transform, registered_input_output_maps): updated_contents.append(updated_content) updated_input_contents_full.append(content) elif self.is_all_inputs_dependency_terminated(inputs_dependency): - self.logger.debug("all input dependency terminated: %s, inputs: %s, outputs: %s" % (str(inputs_dependency), str(inputs), str(outputs))) + # self.logger.debug("all input dependency terminated: %s, inputs: %s, outputs: %s" % (str(inputs_dependency), str(inputs), str(outputs))) for content in inputs: content['substatus'] = ContentStatus.Missing if content['status'] != content['substatus']: @@ -265,28 +265,6 @@ def get_updated_contents(self, transform, registered_input_output_maps): updated_output_contents_full.append(content) return updated_contents, updated_input_contents_full, updated_output_contents_full - def trigger_release_inputs_old(self, updated_output_contents, work, input_output_maps): - to_release_inputs = [] - for content in updated_output_contents: - if (content['status'] in [ContentStatus.Available, ContentStatus.Available.value, ContentStatus.FakeAvailable, ContentStatus.FakeAvailable.value] - or content['substatus'] in [ContentStatus.Available, ContentStatus.Available.value, ContentStatus.FakeAvailable, ContentStatus.FakeAvailable.value]): # noqa W503 - to_release = {'request_id': content['request_id'], - 'coll_id': content['coll_id'], - 'name': content['name'], - 'status': content['status'], - 'substatus': content['substatus']} - to_release_inputs.append(to_release) - - to_release_inputs_backup = work.get_backup_to_release_inputs() - work.add_backup_to_release_inputs(to_release_inputs) - - # updated_contents = core_transforms.release_inputs(to_release_inputs) - self.logger.debug("trigger_release_inputs, to_release_inputs: %s" % str(to_release_inputs)) - self.logger.debug("trigger_release_inputs, to_release_inputs_backup: %s" % str(to_release_inputs_backup)) - updated_contents = core_transforms.release_inputs(to_release_inputs + to_release_inputs_backup) - self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) - return updated_contents - def trigger_release_inputs(self, updated_output_contents, work, input_output_maps, final=False): to_release_inputs = {} for map_id in input_output_maps: @@ -300,8 +278,8 @@ def trigger_release_inputs(self, updated_output_contents, work, input_output_map # updated_contents = core_transforms.release_inputs(to_release_inputs) updated_contents = core_transforms.release_inputs_by_collection(to_release_inputs, final=final) - self.logger.debug("trigger_release_inputs, to_release_inputs: %s" % str(to_release_inputs)) - self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) + # self.logger.debug("trigger_release_inputs, to_release_inputs: %s" % str(to_release_inputs)) + # self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) return updated_contents def poll_inputs_dependency(self, transform, registered_input_output_maps): @@ -317,7 +295,7 @@ def poll_inputs_dependency(self, transform, registered_input_output_maps): # updated_contents = core_transforms.release_inputs(to_release_inputs) updated_contents = core_transforms.poll_inputs_dependency_by_collection(unfinished_inputs) - self.logger.debug("poll_inputs_dependency, updated_contents: %s" % str(updated_contents)) + # self.logger.debug("poll_inputs_dependency, updated_contents: %s" % str(updated_contents)) return updated_contents def process_new_transform_real(self, transform): @@ -873,9 +851,10 @@ def process_running_transform_real(self, transform): # if work.use_dependency_to_release_jobs() and (updated_output_contents_full or work.has_to_release_inputs()): if work.use_dependency_to_release_jobs(): pass - # self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) - # to_release_input_contents = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps) - to_release_input_contents = self.poll_inputs_dependency(transform, registered_input_output_maps) + self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) + to_release_input_contents = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps) + if not to_release_input_contents: + to_release_input_contents = self.poll_inputs_dependency(transform, registered_input_output_maps) self.logger.info("generate_message: %s" % transform['transform_id']) if new_input_contents: @@ -902,9 +881,9 @@ def process_running_transform_real(self, transform): self.logger.info("Transform(%s) work is terminated, trigger to release all final status files" % (transform['transform_id'])) if work.use_dependency_to_release_jobs(): pass - # self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) - # to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps, final=True) - # to_release_input_contents = to_release_input_contents + to_release_input_contents1 + self.logger.info("trigger_release_inputs: %s" % transform['transform_id']) + to_release_input_contents1 = self.trigger_release_inputs(updated_output_contents_full, work, registered_input_output_maps, final=True) + to_release_input_contents = to_release_input_contents + to_release_input_contents1 to_resume_transform = False reactivated_contents = [] From bf1dacc8727cd0f5ae7062494be5f5e4355e5690 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 19 Nov 2021 15:15:18 +0100 Subject: [PATCH 137/156] optimize logging --- doma/lib/idds/doma/workflow/domapandawork.py | 12 ++++++------ doma/lib/idds/doma/workflowv2/domapandawork.py | 12 ++++++------ main/lib/idds/tests/conddor_test.sh | 3 ++- monitor/conf.js | 12 ++++++------ 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index 7ab5a23d..4eb937c0 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -688,7 +688,7 @@ def poll_panda_task(self, processing=None, input_output_maps=None): if task_id: # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) - self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) + # self.logger.debug("poll_panda_task, task_info: %s" % str(task_info)) if task_info[0] != 0: self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) return ProcessingStatus.Submitting, {} @@ -836,7 +836,7 @@ def poll_processing_updates(self, processing, input_output_maps): processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) - self.logger.debug("poll_processing_updates, update_contents: %s" % str(poll_updated_contents)) + self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) if poll_updated_contents: proc.has_new_updates() @@ -890,10 +890,10 @@ def poll_processing_updates(self, processing, input_output_maps): self.logger.debug("poll_processing_updates, task: %s, update_processing: %s" % (proc.workload_id, str(update_processing))) - self.logger.debug("poll_processing_updates, task: %s, updated_contents: %s" % - (proc.workload_id, str(updated_contents))) - self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % - (proc.workload_id, str(reactive_contents))) + self.logger.debug("poll_processing_updates, task: %s, updated_contents[:100]: %s" % + (proc.workload_id, str(updated_contents[:100]))) + self.logger.debug("poll_processing_updates, task: %s, reactive_contents[:100]: %s" % + (proc.workload_id, str(reactive_contents[:100]))) return update_processing, updated_contents + reactive_contents, {} def get_status_statistics(self, registered_input_output_maps): diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index e2f46ff4..d5d459ed 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -693,7 +693,7 @@ def poll_panda_task(self, processing=None, input_output_maps=None): if task_id: # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) - self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) + # self.logger.debug("poll_panda_task, task_info: %s" % str(task_info)) if task_info[0] != 0: self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) return ProcessingStatus.Submitting, {} @@ -841,7 +841,7 @@ def poll_processing_updates(self, processing, input_output_maps): processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) - self.logger.debug("poll_processing_updates, update_contents: %s" % str(poll_updated_contents)) + self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) if poll_updated_contents: proc.has_new_updates() @@ -895,10 +895,10 @@ def poll_processing_updates(self, processing, input_output_maps): self.logger.debug("poll_processing_updates, task: %s, update_processing: %s" % (proc.workload_id, str(update_processing))) - self.logger.debug("poll_processing_updates, task: %s, updated_contents: %s" % - (proc.workload_id, str(updated_contents))) - self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % - (proc.workload_id, str(reactive_contents))) + self.logger.debug("poll_processing_updates, task: %s, updated_contents[:100]: %s" % + (proc.workload_id, str(updated_contents[:100]))) + self.logger.debug("poll_processing_updates, task: %s, reactive_contents[:100]: %s" % + (proc.workload_id, str(reactive_contents[:100]))) return update_processing, updated_contents + reactive_contents, {} def get_status_statistics(self, registered_input_output_maps): diff --git a/main/lib/idds/tests/conddor_test.sh b/main/lib/idds/tests/conddor_test.sh index 58484d4c..ca0d833c 100644 --- a/main/lib/idds/tests/conddor_test.sh +++ b/main/lib/idds/tests/conddor_test.sh @@ -11,4 +11,5 @@ cat <> /tmp/test_condor.jdl queue EOT -condor_submit /tmp/test_condor.jdl +#condor_submit /tmp/test_condor.jdl +condor_submit --debug -name aipanda180.cern.ch -pool aipanda180.cern.ch:9618 /tmp/test_condor.jdl diff --git a/monitor/conf.js b/monitor/conf.js index e559ac83..39e1faff 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus739.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus739.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus739.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus739.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" } From 4c4aa842a53252de0b97a11f0a07f8e41bb8e1df Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 19 Nov 2021 15:57:23 +0100 Subject: [PATCH 138/156] optimize poll_input_dep --- main/lib/idds/core/transforms.py | 37 ++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/main/lib/idds/core/transforms.py b/main/lib/idds/core/transforms.py index 4fd577d6..e7cd2b71 100644 --- a/main/lib/idds/core/transforms.py +++ b/main/lib/idds/core/transforms.py @@ -532,13 +532,38 @@ def poll_inputs_dependency_by_collection(unfinished_inputs): to_release_status[content['name']] = content['status'] elif content['substatus'] in status_to_check: to_release_status[content['name']] = content['substatus'] + + unfinished_contents_dict = {} for content in unfinished_contents: - if content['name'] in to_release_status: - if (content['status'] != to_release_status[content['name']]): - update_content = {'content_id': content['content_id'], - 'substatus': to_release_status[content['name']], - 'status': to_release_status[content['name']]} - update_contents.append(update_content) + unfinished_contents_dict[content['name']] = {'content_id': content['content_id'], 'status': content['status']} + + if len(unfinished_contents_dict.keys()) < len(to_release_status.keys()): + for name, content in unfinished_contents_dict.items(): + if name in to_release_status: + matched_content_status = to_release_status[name] + if (content['status'] != matched_content_status): + update_content = {'content_id': content['content_id'], + 'substatus': matched_content_status, + 'status': matched_content_status} + update_contents.append(update_content) + else: + for key, status in to_release_status.items(): + if name in unfinished_contents_dict: + matched_content = unfinished_contents_dict[name] + if (matched_content['status'] != status): + update_content = {'content_id': matched_content['content_id'], + 'substatus': status, + 'status': status} + update_contents.append(update_content) + + # for content in unfinished_contents: + # if content['name'] in to_release_status: + # matched_content_status = to_release_status[content['name']] + # if (content['status'] != matched_content_status): + # update_content = {'content_id': content['content_id'], + # 'substatus': matched_content_status, + # 'status': matched_content_status} + # update_contents.append(update_content) return update_contents From 4b9d5c52b0fa7adc01cbac87921d8a2a2ee6d165 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 19 Nov 2021 19:56:17 +0100 Subject: [PATCH 139/156] handle deadlock for transformer --- .../idds/agents/transformer/transformer.py | 135 +++++++++++++----- 1 file changed, 96 insertions(+), 39 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 10b7a882..c08b66e6 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -10,6 +10,7 @@ import copy import datetime +import time import traceback try: # python 3 @@ -421,26 +422,54 @@ def process_new_transforms(self): def finish_new_transforms(self): while not self.new_output_queue.empty(): try: - ret = self.new_output_queue.get() - self.logger.info("Main thread finishing processing transform: %s" % ret['transform']) - if ret: - # self.logger.debug("wen: %s" % str(ret['output_contents'])) - core_transforms.add_transform_outputs(transform=ret['transform'], - transform_parameters=ret['transform_parameters'], - input_collections=ret.get('input_collections', None), - output_collections=ret.get('output_collections', None), - log_collections=ret.get('log_collections', None), - new_contents=ret.get('new_contents', None), - update_input_collections=ret.get('update_input_collections', None), - update_output_collections=ret.get('update_output_collections', None), - update_log_collections=ret.get('update_log_collections', None), - update_contents=ret.get('update_contents', None), - messages=ret.get('messages', None), - new_processing=ret.get('new_processing', None), - message_bulk_size=self.message_bulk_size) + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: + ret = self.new_output_queue.get() + self.logger.info("Main thread finishing processing transform: %s" % ret['transform']) + if ret: + # self.logger.debug("wen: %s" % str(ret['output_contents'])) + core_transforms.add_transform_outputs(transform=ret['transform'], + transform_parameters=ret['transform_parameters'], + input_collections=ret.get('input_collections', None), + output_collections=ret.get('output_collections', None), + log_collections=ret.get('log_collections', None), + new_contents=ret.get('new_contents', None), + update_input_collections=ret.get('update_input_collections', None), + update_output_collections=ret.get('update_output_collections', None), + update_log_collections=ret.get('update_log_collections', None), + update_contents=ret.get('update_contents', None), + messages=ret.get('messages', None), + new_processing=ret.get('new_processing', None), + message_bulk_size=self.message_bulk_size) + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex + else: + raise ex + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + try: + transform_parameters = {'status': TransformStatus.Transforming, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period), + 'retries': ret['transform']['retries'] + 1, + 'locking': TransformLocking.Idle} + core_transforms.add_transform_outputs(transform=ret['transform'], + transform_parameters=transform_parameters) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) def get_running_transforms(self): """ @@ -1144,31 +1173,59 @@ def process_running_transforms(self): def finish_running_transforms(self): while not self.running_output_queue.empty(): try: - ret = self.running_output_queue.get() - self.logger.debug("Main thread finishing running transform: %s" % ret['transform']) - self.logger.info("Main thread finishing running transform(%s): %s" % (ret['transform']['transform_id'], - ret['transform_parameters'])) - if ret: - # self.logger.debug("wen: %s" % str(ret['output_contents'])) - core_transforms.add_transform_outputs(transform=ret['transform'], - transform_parameters=ret['transform_parameters'], - input_collections=ret.get('input_collections', None), - output_collections=ret.get('output_collections', None), - log_collections=ret.get('log_collections', None), - new_contents=ret.get('new_contents', None), - update_input_collections=ret.get('update_input_collections', None), - update_output_collections=ret.get('update_output_collections', None), - update_log_collections=ret.get('update_log_collections', None), - update_contents=ret.get('update_contents', None), - messages=ret.get('messages', None), - update_messages=ret.get('update_messages', None), - new_processing=ret.get('new_processing', None), - update_processing=ret.get('update_processing', None), - message_bulk_size=self.message_bulk_size) - + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: + ret = self.running_output_queue.get() + self.logger.debug("Main thread finishing running transform: %s" % ret['transform']) + self.logger.info("Main thread finishing running transform(%s): %s" % (ret['transform']['transform_id'], + ret['transform_parameters'])) + if ret: + # self.logger.debug("wen: %s" % str(ret['output_contents'])) + core_transforms.add_transform_outputs(transform=ret['transform'], + transform_parameters=ret['transform_parameters'], + input_collections=ret.get('input_collections', None), + output_collections=ret.get('output_collections', None), + log_collections=ret.get('log_collections', None), + new_contents=ret.get('new_contents', None), + update_input_collections=ret.get('update_input_collections', None), + update_output_collections=ret.get('update_output_collections', None), + update_log_collections=ret.get('update_log_collections', None), + update_contents=ret.get('update_contents', None), + messages=ret.get('messages', None), + update_messages=ret.get('update_messages', None), + new_processing=ret.get('new_processing', None), + update_processing=ret.get('update_processing', None), + message_bulk_size=self.message_bulk_size) + + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex + else: + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) + raise ex except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + try: + transform_parameters = {'status': TransformStatus.Transforming, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period), + 'retries': ret['transform']['retries'] + 1, + 'locking': TransformLocking.Idle} + core_transforms.add_transform_outputs(transform=ret['transform'], + transform_parameters=transform_parameters) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) def clean_locks(self): self.logger.info("clean locking") From 0ec8b55a545d9a86ab4e4a5b199af904331f56f4 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 19 Nov 2021 20:44:58 +0100 Subject: [PATCH 140/156] optimize retries for deadlocks --- main/lib/idds/agents/carrier/carrier.py | 65 +++++++++++++-- main/lib/idds/agents/clerk/clerk.py | 71 +++++++++++++--- .../idds/agents/transformer/transformer.py | 80 +++++++++---------- 3 files changed, 161 insertions(+), 55 deletions(-) diff --git a/main/lib/idds/agents/carrier/carrier.py b/main/lib/idds/agents/carrier/carrier.py index c88b0430..a8604c2a 100644 --- a/main/lib/idds/agents/carrier/carrier.py +++ b/main/lib/idds/agents/carrier/carrier.py @@ -6,9 +6,10 @@ # http://www.apache.org/licenses/LICENSE-2.0OA # # Authors: -# - Wen Guan, , 2019 - 2020 +# - Wen Guan, , 2019 - 2021 import datetime +import time import traceback try: # python 3 @@ -148,10 +149,36 @@ def finish_new_processings(self): del processing['processing_id'] processing['locking'] = ProcessingLocking.Idle # self.logger.debug("wen: %s" % str(processing)) - core_processings.update_processing(processing_id=processing_id, parameters=processing) + + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: + core_processings.update_processing(processing_id=processing_id, parameters=processing) + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex + else: + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) + raise ex except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + try: + parameters = {'status': ProcessingStatus.Running, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4)} + core_processings.update_processing(processing_id=processing_id, parameters=parameters) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) def get_running_processings(self): """ @@ -474,13 +501,39 @@ def finish_running_processings(self): processing['processing_update']['parameters'])) # self.logger.info("Main thread finishing running processing %s" % str(processing)) - core_processings.update_processing_contents(processing_update=processing.get('processing_update', None), - content_updates=processing.get('content_updates', None), - update_messages=processing.get('update_messages', None), - new_contents=processing.get('new_contents', None)) + + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: + core_processings.update_processing_contents(processing_update=processing.get('processing_update', None), + content_updates=processing.get('content_updates', None), + update_messages=processing.get('update_messages', None), + new_contents=processing.get('new_contents', None)) + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex + else: + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) + raise ex except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + try: + parameters = {'status': ProcessingStatus.Running, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period * 4)} + core_processings.update_processing(processing_id=processing['processing_update']['processing_id'], parameters=parameters) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) def clean_locks(self): self.logger.info("clean locking") diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index 31354ed9..566717c1 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -9,6 +9,7 @@ # - Wen Guan, , 2019 - 2021 import datetime +import time import traceback try: # python 3 @@ -204,12 +205,38 @@ def finish_new_requests(self): else: update_transforms = {} - core_requests.update_request_with_transforms(req['request_id'], req['parameters'], - new_transforms=new_transforms, - update_transforms=update_transforms) + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: + core_requests.update_request_with_transforms(req['request_id'], req['parameters'], + new_transforms=new_transforms, + update_transforms=update_transforms) + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex + else: + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) + raise ex except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + try: + req_parameters = {'status': RequestStatus.Transforming, + 'locking': RequestLocking.Idle, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period)} + core_requests.update_request_with_transforms(req['request_id'], req_parameters) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) def get_running_requests(self): """ @@ -541,15 +568,41 @@ def finish_running_requests(self): else: update_transforms = {} - core_requests.update_request_with_transforms(req['request_id'], req['parameters'], - new_transforms=new_transforms, - update_transforms=update_transforms, - new_messages=req.get('new_messages', None), - update_messages=req.get('update_messages', None)) - + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: + core_requests.update_request_with_transforms(req['request_id'], req['parameters'], + new_transforms=new_transforms, + update_transforms=update_transforms, + new_messages=req.get('new_messages', None), + update_messages=req.get('update_messages', None)) + + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex + else: + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) + raise ex except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) + try: + req_parameters = {'status': RequestStatus.Transforming, + 'locking': RequestLocking.Idle, + 'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period)} + core_requests.update_request_with_transforms(req['request_id'], req_parameters) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) def clean_locks(self): self.logger.info("clean locking") diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index c08b66e6..259df08a 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -422,15 +422,15 @@ def process_new_transforms(self): def finish_new_transforms(self): while not self.new_output_queue.empty(): try: - retry = True - retry_num = 0 - while retry: - retry = False - retry_num += 1 - try: - ret = self.new_output_queue.get() - self.logger.info("Main thread finishing processing transform: %s" % ret['transform']) - if ret: + ret = self.new_output_queue.get() + self.logger.info("Main thread finishing processing transform: %s" % ret['transform']) + if ret: + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: # self.logger.debug("wen: %s" % str(ret['output_contents'])) core_transforms.add_transform_outputs(transform=ret['transform'], transform_parameters=ret['transform_parameters'], @@ -445,18 +445,18 @@ def finish_new_transforms(self): messages=ret.get('messages', None), new_processing=ret.get('new_processing', None), message_bulk_size=self.message_bulk_size) - except exceptions.DatabaseException as ex: - if 'ORA-00060' in str(ex): - self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") - if retry_num < 5: - retry = True - time.sleep(60 * retry_num * 2) + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex else: raise ex - else: - raise ex - # self.logger.error(ex) - # self.logger.error(traceback.format_exc()) + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) @@ -1173,17 +1173,17 @@ def process_running_transforms(self): def finish_running_transforms(self): while not self.running_output_queue.empty(): try: - retry = True - retry_num = 0 - while retry: - retry = False - retry_num += 1 - try: - ret = self.running_output_queue.get() - self.logger.debug("Main thread finishing running transform: %s" % ret['transform']) - self.logger.info("Main thread finishing running transform(%s): %s" % (ret['transform']['transform_id'], - ret['transform_parameters'])) - if ret: + ret = self.running_output_queue.get() + self.logger.debug("Main thread finishing running transform: %s" % ret['transform']) + self.logger.info("Main thread finishing running transform(%s): %s" % (ret['transform']['transform_id'], + ret['transform_parameters'])) + if ret: + retry = True + retry_num = 0 + while retry: + retry = False + retry_num += 1 + try: # self.logger.debug("wen: %s" % str(ret['output_contents'])) core_transforms.add_transform_outputs(transform=ret['transform'], transform_parameters=ret['transform_parameters'], @@ -1201,18 +1201,18 @@ def finish_running_transforms(self): update_processing=ret.get('update_processing', None), message_bulk_size=self.message_bulk_size) - except exceptions.DatabaseException as ex: - if 'ORA-00060' in str(ex): - self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") - if retry_num < 5: - retry = True - time.sleep(60 * retry_num * 2) + except exceptions.DatabaseException as ex: + if 'ORA-00060' in str(ex): + self.logger.warn("(cx_Oracle.DatabaseError) ORA-00060: deadlock detected while waiting for resource") + if retry_num < 5: + retry = True + time.sleep(60 * retry_num * 2) + else: + raise ex else: + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) raise ex - else: - # self.logger.error(ex) - # self.logger.error(traceback.format_exc()) - raise ex except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) From 7d0bf839c35a8b1122a18cacbea828e8dccd30f0 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 19 Nov 2021 23:49:08 +0100 Subject: [PATCH 141/156] fix log --- doma/lib/idds/doma/workflow/domapandawork.py | 2 +- doma/lib/idds/doma/workflowv2/domapandawork.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index 4eb937c0..e09b3112 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -836,7 +836,7 @@ def poll_processing_updates(self, processing, input_output_maps): processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) - self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) + # self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) if poll_updated_contents: proc.has_new_updates() diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index d5d459ed..a42763c1 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -841,7 +841,7 @@ def poll_processing_updates(self, processing, input_output_maps): processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) - self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) + # self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) if poll_updated_contents: proc.has_new_updates() From eca1844e39ebae2dfb06530e73ab3f845815f6c7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 20 Nov 2021 11:44:54 +0100 Subject: [PATCH 142/156] fix poll input depdendencies --- main/lib/idds/core/transforms.py | 2 +- main/lib/idds/tests/panda_test.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/main/lib/idds/core/transforms.py b/main/lib/idds/core/transforms.py index e7cd2b71..6906d816 100644 --- a/main/lib/idds/core/transforms.py +++ b/main/lib/idds/core/transforms.py @@ -547,7 +547,7 @@ def poll_inputs_dependency_by_collection(unfinished_inputs): 'status': matched_content_status} update_contents.append(update_content) else: - for key, status in to_release_status.items(): + for name, status in to_release_status.items(): if name in unfinished_contents_dict: matched_content = unfinished_contents_dict[name] if (matched_content['status'] != status): diff --git a/main/lib/idds/tests/panda_test.py b/main/lib/idds/tests/panda_test.py index 8cfa1fe4..8b59f959 100644 --- a/main/lib/idds/tests/panda_test.py +++ b/main/lib/idds/tests/panda_test.py @@ -27,9 +27,9 @@ print(f._attributes) print(f.values()) print(f.type) +""" - -jediTaskID = 3885 +jediTaskID = 7784 ret = Client.getJediTaskDetails({'jediTaskID': jediTaskID}, True, True, verbose=False) print(ret) @@ -38,6 +38,7 @@ sys.exit(0) +""" jediTaskID = 998 ret = Client.getPandaIDsWithTaskID(jediTaskID, verbose=False) # print(ret) From 41e4f708177c3d2301bdd88805d993d20d422430 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 22 Nov 2021 15:37:20 +0100 Subject: [PATCH 143/156] add condor sec conf --- main/etc/condor/collector/09_flin_condor.config | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 main/etc/condor/collector/09_flin_condor.config diff --git a/main/etc/condor/collector/09_flin_condor.config b/main/etc/condor/collector/09_flin_condor.config new file mode 100644 index 00000000..cb8c3fee --- /dev/null +++ b/main/etc/condor/collector/09_flin_condor.config @@ -0,0 +1,3 @@ +SCHEDD_DEBUG = D_FULLDEBUG, D_SECURITY + +SEC_DEFAULT_AUTHENTICATION_METHODS = CLAIMTOBE, $(SEC_DEFAULT_AUTHENTICATION_METHODS) From d8a4b8f34384e6ead4850757fcce13a893540a2d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 26 Nov 2021 14:18:53 +0100 Subject: [PATCH 144/156] use intersection to release jobs --- .../idds/agents/transformer/transformer.py | 4 +- main/lib/idds/core/transforms.py | 87 +++++++++++++------ main/lib/idds/tests/core_tests.py | 6 +- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/main/lib/idds/agents/transformer/transformer.py b/main/lib/idds/agents/transformer/transformer.py index 259df08a..ecc1cfe0 100644 --- a/main/lib/idds/agents/transformer/transformer.py +++ b/main/lib/idds/agents/transformer/transformer.py @@ -280,7 +280,7 @@ def trigger_release_inputs(self, updated_output_contents, work, input_output_map # updated_contents = core_transforms.release_inputs(to_release_inputs) updated_contents = core_transforms.release_inputs_by_collection(to_release_inputs, final=final) # self.logger.debug("trigger_release_inputs, to_release_inputs: %s" % str(to_release_inputs)) - # self.logger.debug("trigger_release_inputs, updated_contents: %s" % str(updated_contents)) + self.logger.debug("trigger_release_inputs, updated_contents[:10]: %s" % str(updated_contents[:10])) return updated_contents def poll_inputs_dependency(self, transform, registered_input_output_maps): @@ -296,7 +296,7 @@ def poll_inputs_dependency(self, transform, registered_input_output_maps): # updated_contents = core_transforms.release_inputs(to_release_inputs) updated_contents = core_transforms.poll_inputs_dependency_by_collection(unfinished_inputs) - # self.logger.debug("poll_inputs_dependency, updated_contents: %s" % str(updated_contents)) + self.logger.debug("poll_inputs_dependency, updated_contents[:10]: %s" % str(updated_contents[:10])) return updated_contents def process_new_transform_real(self, transform): diff --git a/main/lib/idds/core/transforms.py b/main/lib/idds/core/transforms.py index 6906d816..98006dea 100644 --- a/main/lib/idds/core/transforms.py +++ b/main/lib/idds/core/transforms.py @@ -13,6 +13,8 @@ operations related to Transform. """ +import logging + # from idds.common import exceptions from idds.common.constants import (TransformStatus, ContentRelationType, ContentStatus, @@ -504,14 +506,28 @@ def release_inputs_by_collection(to_release_inputs, final=False): name=None) # print("contents: %s" % str(contents)) + unfinished_contents_dict = {} for content in contents: if (content['content_relation_type'] == ContentRelationType.InputDependency): # noqa: W503 - if content['name'] in to_release_status: - if final or (content['status'] != to_release_status[content['name']]): - update_content = {'content_id': content['content_id'], - 'substatus': to_release_status[content['name']], - 'status': to_release_status[content['name']]} - update_contents.append(update_content) + if content['status'] not in status_to_check: + if content['name'] not in unfinished_contents_dict: + unfinished_contents_dict[content['name']] = [] + content_short = {'content_id': content['content_id'], 'status': content['status']} + unfinished_contents_dict[content['name']].append(content_short) + + intersection_keys = to_release_status.keys() & unfinished_contents_dict.keys() + intersection_keys = list(intersection_keys) + logging.debug("release_inputs_by_collection(coll_id: %s): intersection_keys[:10]: %s" % (coll_id, str(intersection_keys[:10]))) + + for name in intersection_keys: + matched_content_status = to_release_status[name] + matched_contents = unfinished_contents_dict[name] + for matched_content in matched_contents: + if (matched_content['status'] != matched_content_status): + update_content = {'content_id': matched_content['content_id'], + 'substatus': matched_content_status, + 'status': matched_content_status} + update_contents.append(update_content) return update_contents @@ -525,6 +541,8 @@ def poll_inputs_dependency_by_collection(unfinished_inputs): coll_id=unfinished_contents[0]['coll_id'], name=None) + logging.debug("poll_inputs_dependency_by_collection(coll_id: %s): unfinished_contents[:10]: %s" % (coll_id, str(unfinished_contents[:10]))) + to_release_status = {} for content in contents: if (content['content_relation_type'] == ContentRelationType.Output): # noqa: W503 @@ -535,26 +553,43 @@ def poll_inputs_dependency_by_collection(unfinished_inputs): unfinished_contents_dict = {} for content in unfinished_contents: - unfinished_contents_dict[content['name']] = {'content_id': content['content_id'], 'status': content['status']} - - if len(unfinished_contents_dict.keys()) < len(to_release_status.keys()): - for name, content in unfinished_contents_dict.items(): - if name in to_release_status: - matched_content_status = to_release_status[name] - if (content['status'] != matched_content_status): - update_content = {'content_id': content['content_id'], - 'substatus': matched_content_status, - 'status': matched_content_status} - update_contents.append(update_content) - else: - for name, status in to_release_status.items(): - if name in unfinished_contents_dict: - matched_content = unfinished_contents_dict[name] - if (matched_content['status'] != status): - update_content = {'content_id': matched_content['content_id'], - 'substatus': status, - 'status': status} - update_contents.append(update_content) + if content['name'] not in unfinished_contents_dict: + unfinished_contents_dict[content['name']] = [] + content_short = {'content_id': content['content_id'], 'status': content['status']} + unfinished_contents_dict[content['name']].append(content_short) + + intersection_keys = to_release_status.keys() & unfinished_contents_dict.keys() + intersection_keys = list(intersection_keys) + logging.debug("poll_inputs_dependency_by_collection(coll_id: %s): intersection_keys[:10]: %s" % (coll_id, str(intersection_keys[:10]))) + + for name in intersection_keys: + matched_content_status = to_release_status[name] + matched_contents = unfinished_contents_dict[name] + for matched_content in matched_contents: + if (matched_content['status'] != matched_content_status): + update_content = {'content_id': matched_content['content_id'], + 'substatus': matched_content_status, + 'status': matched_content_status} + update_contents.append(update_content) + + # if len(unfinished_contents_dict.keys()) < len(to_release_status.keys()): + # for name, content in unfinished_contents_dict.items(): + # if name in to_release_status: + # matched_content_status = to_release_status[name] + # if (content['status'] != matched_content_status): + # update_content = {'content_id': content['content_id'], + # 'substatus': matched_content_status, + # 'status': matched_content_status} + # update_contents.append(update_content) + # else: + # for name, status in to_release_status.items(): + # if name in unfinished_contents_dict: + # matched_content = unfinished_contents_dict[name] + # if (matched_content['status'] != status): + # update_content = {'content_id': matched_content['content_id'], + # 'substatus': status, + # 'status': status} + # update_contents.append(update_content) # for content in unfinished_contents: # if content['name'] in to_release_status: diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 80ad550a..58f0bccc 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,15 +102,15 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=219, with_detail=True, with_metadata=True) +reqs = get_requests(request_id=538, with_detail=True, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) - # print(json_dumps(req, sort_keys=True, indent=4)) + print(json_dumps(req, sort_keys=True, indent=4)) # show_works(req) pass -# sys.exit(0) +sys.exit(0) """ # reqs = get_requests() From 711dd4ff0a7737baa97297c756ce360edf49f223 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 26 Nov 2021 14:48:58 +0100 Subject: [PATCH 145/156] optimize logs --- doma/lib/idds/doma/workflow/domapandawork.py | 4 ++-- doma/lib/idds/doma/workflowv2/domapandawork.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index e09b3112..ba508502 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -618,7 +618,7 @@ def get_update_contents_from_map_id(self, map_id, input_output_maps, job_info): return update_contents def map_panda_ids(self, unregistered_job_ids, input_output_maps): - self.logger.debug("map_panda_ids, unregistered_job_ids: %s" % str(unregistered_job_ids)) + self.logger.debug("map_panda_ids, unregistered_job_ids[:10]: %s" % str(unregistered_job_ids[:10])) from pandatools import Client # updated_map_ids = [] @@ -646,7 +646,7 @@ def map_panda_ids(self, unregistered_job_ids, input_output_maps): return full_update_contents def get_status_changed_contents(self, unterminated_job_ids, input_output_maps, panda_id_to_map_ids): - self.logger.debug("get_status_changed_contents, unterminated_job_ids: %s" % str(unterminated_job_ids)) + self.logger.debug("get_status_changed_contents, unterminated_job_ids[:10]: %s" % str(unterminated_job_ids[:10])) from pandatools import Client full_update_contents = [] diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index a42763c1..b7661b32 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -623,7 +623,7 @@ def get_update_contents_from_map_id(self, map_id, input_output_maps, job_info): return update_contents def map_panda_ids(self, unregistered_job_ids, input_output_maps): - self.logger.debug("map_panda_ids, unregistered_job_ids: %s" % str(unregistered_job_ids)) + self.logger.debug("map_panda_ids, unregistered_job_ids[:10]: %s" % str(unregistered_job_ids[:10])) from pandatools import Client # updated_map_ids = [] @@ -651,7 +651,7 @@ def map_panda_ids(self, unregistered_job_ids, input_output_maps): return full_update_contents def get_status_changed_contents(self, unterminated_job_ids, input_output_maps, panda_id_to_map_ids): - self.logger.debug("get_status_changed_contents, unterminated_job_ids: %s" % str(unterminated_job_ids)) + self.logger.debug("get_status_changed_contents, unterminated_job_ids[:10]: %s" % str(unterminated_job_ids[:10])) from pandatools import Client full_update_contents = [] From b9dcb3c3a21fcac638ab36c7b789323603a9cdf4 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 29 Nov 2021 13:45:45 +0100 Subject: [PATCH 146/156] optimize doma work on polling panda jobs --- doma/lib/idds/doma/workflow/domapandawork.py | 207 +++++++++++++++++- .../lib/idds/doma/workflowv2/domapandawork.py | 207 +++++++++++++++++- 2 files changed, 404 insertions(+), 10 deletions(-) diff --git a/doma/lib/idds/doma/workflow/domapandawork.py b/doma/lib/idds/doma/workflow/domapandawork.py index ba508502..7c733130 100644 --- a/doma/lib/idds/doma/workflow/domapandawork.py +++ b/doma/lib/idds/doma/workflow/domapandawork.py @@ -86,6 +86,8 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.retry_number = 0 self.num_retries = num_retries + self.poll_panda_jobs_chunk_size = 2000 + self.load_panda_urls() def my_condition(self): @@ -145,6 +147,8 @@ def set_agent_attributes(self, attrs, req_attributes=None): super(DomaPanDAWork, self).set_agent_attributes(attrs) if 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: self.num_retries = int(self.agent_attributes['num_retries']) + if 'poll_panda_jobs_chunk_size' in self.agent_attributes and self.agent_attributes['poll_panda_jobs_chunk_size']: + self.poll_panda_jobs_chunk_size = int(self.agent_attributes['poll_panda_jobs_chunk_size']) def depend_on(self, work): for job in self.dependency_map: @@ -412,7 +416,7 @@ def submit_panda_task(self, processing): proc = processing['processing_metadata']['processing'] task_param = proc.processing_metadata['task_param'] - return_code = Client.insertTaskParams(task_param, verbose=True) + return_code = Client.insertTaskParams(task_param, verbose=False) if return_code[0] == 0: return return_code[1][1] else: @@ -673,7 +677,7 @@ def get_final_update_contents(self, input_output_maps): return update_contents - def poll_panda_task(self, processing=None, input_output_maps=None): + def poll_panda_task_old(self, processing=None, input_output_maps=None): task_id = None try: from pandatools import Client @@ -687,8 +691,9 @@ def poll_panda_task(self, processing=None, input_output_maps=None): if task_id: # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + self.logger.debug("poll_panda_task, task_id: %s" % str(task_id)) task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) - # self.logger.debug("poll_panda_task, task_info: %s" % str(task_info)) + self.logger.debug("poll_panda_task, task_info[0]: %s" % str(task_info[0])) if task_info[0] != 0: self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) return ProcessingStatus.Submitting, {} @@ -743,6 +748,198 @@ def poll_panda_task(self, processing=None, input_output_maps=None): # raise exceptions.IDDSException(msg) return ProcessingStatus.Submitting, [] + def poll_panda_jobs(self, job_ids): + job_ids = list(job_ids) + self.logger.debug("poll_panda_jobs, poll_panda_jobs_chunk_size: %s, job_ids[:10]: %s" % (self.poll_panda_jobs_chunk_size, str(job_ids[:10]))) + from pandatools import Client + + # updated_map_ids = [] + inputname_jobid_map = {} + chunksize = self.poll_panda_jobs_chunk_size + chunks = [job_ids[i:i + chunksize] for i in range(0, len(job_ids), chunksize)] + for chunk in chunks: + jobs_list = Client.getJobStatus(chunk, verbose=0)[1] + self.logger.debug("poll_panda_jobs, input jobs: %s, output_jobs: %s" % (len(chunk), len(jobs_list))) + for job_info in jobs_list: + job_status = self.get_content_status_from_panda_status(job_info) + if job_info and job_info.Files and len(job_info.Files) > 0: + for job_file in job_info.Files: + # if job_file.type in ['log']: + if job_file.type not in ['pseudo_input']: + continue + if ':' in job_file.lfn: + pos = job_file.lfn.find(":") + input_file = job_file.lfn[pos + 1:] + # input_file = job_file.lfn.split(':')[1] + else: + input_file = job_file.lfn + inputname_jobid_map[input_file] = {'panda_id': job_info.PandaID, 'status': job_status} + return inputname_jobid_map + + def get_job_maps(self, input_output_maps): + inputname_mapid_map = {} + finished_jobs, failed_jobs = [], [] + for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] + outputs = input_output_maps[map_id]['outputs'] + outputs_short = [] + for content in outputs: + outputs_short.append({'content_id': content['content_id'], + 'status': content['status'], + 'substatus': content['substatus'], + 'content_metadata': content['content_metadata']}) + + if content['status'] in [ContentStatus.Available]: + if 'panda_id' in content['content_metadata']: + finished_jobs.append(content['content_metadata']['panda_id']) + elif content['status'] in [ContentStatus.Failed, ContentStatus.FinalFailed, + ContentStatus.Lost, ContentStatus.Deleted, + ContentStatus.Missing]: + if 'panda_id' in content['content_metadata']: + failed_jobs.append(content['content_metadata']['panda_id']) + for content in inputs: + inputname_mapid_map[content['name']] = {'map_id': map_id, + 'outputs': outputs_short} + return finished_jobs + failed_jobs, inputname_mapid_map + + def get_update_contents(self, inputnames, inputname_mapid_map, inputname_jobid_map): + self.logger.debug("get_update_contents, inputnames[:5]: %s" % str(inputnames[:5])) + update_contents = [] + num_updated_contents, num_unupdated_contents = 0, 0 + for inputname in inputnames: + panda_id_status = inputname_jobid_map[inputname] + panda_id = panda_id_status['panda_id'] + panda_status = panda_id_status['status'] + map_id_contents = inputname_mapid_map[inputname] + contents = map_id_contents['outputs'] + for content in contents: + if content['substatus'] != panda_status: + if 'panda_id' in content['content_metadata'] and content['content_metadata']['panda_id']: + # if content['content_metadata']['panda_id'] != job_info.PandaID: + if content['content_metadata']['panda_id'] < panda_id: + # new panda id is the bigger one. + if 'old_panda_id' not in content['content_metadata']: + content['content_metadata']['old_panda_id'] = [] + if content['content_metadata']['panda_id'] not in content['content_metadata']['old_panda_id']: + content['content_metadata']['old_panda_id'].append(content['content_metadata']['panda_id']) + content['content_metadata']['panda_id'] = panda_id + content['substatus'] = panda_status + elif content['content_metadata']['panda_id'] > panda_id: + if 'old_panda_id' not in content['content_metadata']: + content['content_metadata']['old_panda_id'] = [] + if panda_id not in content['content_metadata']['old_panda_id']: + content['content_metadata']['old_panda_id'].append(panda_id) + # content['content_metadata']['panda_id'] = content['content_metadata']['panda_id'] + # content['substatus'] = panda_status + else: + pass + # content['content_metadata']['panda_id'] = panda_id + content['substatus'] = panda_status + + update_contents.append(content) + num_updated_contents += 1 + else: + num_unupdated_contents += 1 + self.logger.debug("get_update_contents, num_updated_contents: %s, num_unupdated_contents: %s" % (num_updated_contents, num_unupdated_contents)) + return update_contents + + def poll_panda_task(self, processing=None, input_output_maps=None): + task_id = None + try: + from pandatools import Client + + if processing: + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + if task_id is None: + task_id = self.get_panda_task_id(processing) + + if task_id: + # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + self.logger.debug("poll_panda_task, task_id: %s" % str(task_id)) + task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) + self.logger.debug("poll_panda_task, task_info[0]: %s" % str(task_info[0])) + if task_info[0] != 0: + self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) + return ProcessingStatus.Submitting, {} + + task_info = task_info[1] + + processing_status = self.get_processing_status_from_panda_status(task_info["status"]) + + if processing_status in [ProcessingStatus.SubFinished]: + if self.retry_number < self.num_retries: + self.reactivate_processing(processing) + processing_status = ProcessingStatus.Submitted + self.retry_number += 1 + + all_jobs_ids = task_info['PandaID'] + + terminated_jobs, inputname_mapid_map = self.get_job_maps(input_output_maps) + self.logger.debug("poll_panda_task, task_id: %s, all jobs: %s, terminated_jobs: %s" % (str(task_id), len(all_jobs_ids), len(terminated_jobs))) + + all_jobs_ids = set(all_jobs_ids) + terminated_jobs = set(terminated_jobs) + unterminated_jobs = all_jobs_ids - terminated_jobs + + inputname_jobid_map = self.poll_panda_jobs(unterminated_jobs) + intersection_keys = set(inputname_mapid_map.keys()) & set(inputname_jobid_map.keys()) + + updated_contents = self.get_update_contents(list(intersection_keys), inputname_mapid_map, inputname_jobid_map) + + final_update_contents = [] + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if updated_contents: + # there are still polling contents, should not terminate the task. + log_warn = "Processing (%s) with panda task id (%s) is %s, however there are still updated_contents[:5]: %s" % (processing['processing_id'], + task_id, + processing_status, + str(updated_contents[:5])) + log_warn = log_warn + ". Keep the processing status as running now." + self.logger.warn(log_warn) + processing_status = ProcessingStatus.Running + elif list(unterminated_jobs): + log_warn = "Processing (%s) with panda task id (%s) is %s, however there are still unterminated_jobs[:5]: %s" % (processing['processing_id'], + task_id, + processing_status, + str(list(unterminated_jobs)[:5])) + log_warn = log_warn + ". Keep the processing status as running now." + self.logger.warn(log_warn) + processing_status = ProcessingStatus.Running + else: + # unsubmitted_inputnames = set(inputname_mapid_map.keys()) - set(inputname_jobid_map.keys()) + # unsubmitted_inputnames = list(unsubmitted_inputnames) + # if unsubmitted_inputnames: + # log_warn = "Processing (%s) with panda task id (%s) is %s, however there are still unsubmitted_inputnames[:5]: %s" % (processing['processing_id'], + # task_id, + # processing_status, + # str(unsubmitted_inputnames[:5])) + # log_warn = log_warn + ". Keep the processing status as running now." + # self.logger.warn(log_warn) + # processing_status = ProcessingStatus.Running + + for inputname in inputname_mapid_map: + map_id_contents = inputname_mapid_map[inputname] + contents = map_id_contents['outputs'] + for content in contents: + if (content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed]): + content['content_metadata']['old_final_status'] = content['substatus'] + content['substatus'] = ContentStatus.FinalFailed + # final_update_contents.append(content) # TODO: mark other contents to Missing + + if final_update_contents: + processing_status = ProcessingStatus.Running + return processing_status, updated_contents + final_update_contents + else: + return ProcessingStatus.Failed, [] + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + self.logger.error(msg) + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException(msg) + return ProcessingStatus.Submitting, [] + def kill_processing(self, processing): try: if processing: @@ -794,7 +991,7 @@ def poll_processing_updates(self, processing, input_output_maps): update_processing = {} reset_expired_at = False reactive_contents = [] - # self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) + self.logger.debug("poll_processing_updates, input_output_maps.keys[:5]: %s" % str(list(input_output_maps.keys())[:5])) if processing: proc = processing['processing_metadata']['processing'] @@ -836,7 +1033,7 @@ def poll_processing_updates(self, processing, input_output_maps): processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) - # self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) + self.logger.debug("poll_processing_updates, update_contents[:10]: %s" % str(poll_updated_contents[:10])) if poll_updated_contents: proc.has_new_updates() diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index b7661b32..dab509b0 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -91,6 +91,8 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.retry_number = 0 self.num_retries = num_retries + self.poll_panda_jobs_chunk_size = 2000 + self.load_panda_urls() def my_condition(self): @@ -150,6 +152,8 @@ def set_agent_attributes(self, attrs, req_attributes=None): super(DomaPanDAWork, self).set_agent_attributes(attrs) if 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: self.num_retries = int(self.agent_attributes['num_retries']) + if 'poll_panda_jobs_chunk_size' in self.agent_attributes and self.agent_attributes['poll_panda_jobs_chunk_size']: + self.poll_panda_jobs_chunk_size = int(self.agent_attributes['poll_panda_jobs_chunk_size']) def depend_on(self, work): for job in self.dependency_map: @@ -417,7 +421,7 @@ def submit_panda_task(self, processing): proc = processing['processing_metadata']['processing'] task_param = proc.processing_metadata['task_param'] - return_code = Client.insertTaskParams(task_param, verbose=True) + return_code = Client.insertTaskParams(task_param, verbose=False) if return_code[0] == 0: return return_code[1][1] else: @@ -678,7 +682,7 @@ def get_final_update_contents(self, input_output_maps): return update_contents - def poll_panda_task(self, processing=None, input_output_maps=None): + def poll_panda_task_old(self, processing=None, input_output_maps=None): task_id = None try: from pandatools import Client @@ -692,8 +696,9 @@ def poll_panda_task(self, processing=None, input_output_maps=None): if task_id: # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + self.logger.debug("poll_panda_task, task_id: %s" % str(task_id)) task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) - # self.logger.debug("poll_panda_task, task_info: %s" % str(task_info)) + self.logger.debug("poll_panda_task, task_info[0]: %s" % str(task_info[0])) if task_info[0] != 0: self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) return ProcessingStatus.Submitting, {} @@ -748,6 +753,198 @@ def poll_panda_task(self, processing=None, input_output_maps=None): # raise exceptions.IDDSException(msg) return ProcessingStatus.Submitting, [] + def poll_panda_jobs(self, job_ids): + job_ids = list(job_ids) + self.logger.debug("poll_panda_jobs, poll_panda_jobs_chunk_size: %s, job_ids[:10]: %s" % (self.poll_panda_jobs_chunk_size, str(job_ids[:10]))) + from pandatools import Client + + # updated_map_ids = [] + inputname_jobid_map = {} + chunksize = self.poll_panda_jobs_chunk_size + chunks = [job_ids[i:i + chunksize] for i in range(0, len(job_ids), chunksize)] + for chunk in chunks: + jobs_list = Client.getJobStatus(chunk, verbose=0)[1] + self.logger.debug("poll_panda_jobs, input jobs: %s, output_jobs: %s" % (len(chunk), len(jobs_list))) + for job_info in jobs_list: + job_status = self.get_content_status_from_panda_status(job_info) + if job_info and job_info.Files and len(job_info.Files) > 0: + for job_file in job_info.Files: + # if job_file.type in ['log']: + if job_file.type not in ['pseudo_input']: + continue + if ':' in job_file.lfn: + pos = job_file.lfn.find(":") + input_file = job_file.lfn[pos + 1:] + # input_file = job_file.lfn.split(':')[1] + else: + input_file = job_file.lfn + inputname_jobid_map[input_file] = {'panda_id': job_info.PandaID, 'status': job_status} + return inputname_jobid_map + + def get_job_maps(self, input_output_maps): + inputname_mapid_map = {} + finished_jobs, failed_jobs = [], [] + for map_id in input_output_maps: + inputs = input_output_maps[map_id]['inputs'] + outputs = input_output_maps[map_id]['outputs'] + outputs_short = [] + for content in outputs: + outputs_short.append({'content_id': content['content_id'], + 'status': content['status'], + 'substatus': content['substatus'], + 'content_metadata': content['content_metadata']}) + + if content['status'] in [ContentStatus.Available]: + if 'panda_id' in content['content_metadata']: + finished_jobs.append(content['content_metadata']['panda_id']) + elif content['status'] in [ContentStatus.Failed, ContentStatus.FinalFailed, + ContentStatus.Lost, ContentStatus.Deleted, + ContentStatus.Missing]: + if 'panda_id' in content['content_metadata']: + failed_jobs.append(content['content_metadata']['panda_id']) + for content in inputs: + inputname_mapid_map[content['name']] = {'map_id': map_id, + 'outputs': outputs_short} + return finished_jobs + failed_jobs, inputname_mapid_map + + def get_update_contents(self, inputnames, inputname_mapid_map, inputname_jobid_map): + self.logger.debug("get_update_contents, inputnames[:5]: %s" % str(inputnames[:5])) + update_contents = [] + num_updated_contents, num_unupdated_contents = 0, 0 + for inputname in inputnames: + panda_id_status = inputname_jobid_map[inputname] + panda_id = panda_id_status['panda_id'] + panda_status = panda_id_status['status'] + map_id_contents = inputname_mapid_map[inputname] + contents = map_id_contents['outputs'] + for content in contents: + if content['substatus'] != panda_status: + if 'panda_id' in content['content_metadata'] and content['content_metadata']['panda_id']: + # if content['content_metadata']['panda_id'] != job_info.PandaID: + if content['content_metadata']['panda_id'] < panda_id: + # new panda id is the bigger one. + if 'old_panda_id' not in content['content_metadata']: + content['content_metadata']['old_panda_id'] = [] + if content['content_metadata']['panda_id'] not in content['content_metadata']['old_panda_id']: + content['content_metadata']['old_panda_id'].append(content['content_metadata']['panda_id']) + content['content_metadata']['panda_id'] = panda_id + content['substatus'] = panda_status + elif content['content_metadata']['panda_id'] > panda_id: + if 'old_panda_id' not in content['content_metadata']: + content['content_metadata']['old_panda_id'] = [] + if panda_id not in content['content_metadata']['old_panda_id']: + content['content_metadata']['old_panda_id'].append(panda_id) + # content['content_metadata']['panda_id'] = content['content_metadata']['panda_id'] + # content['substatus'] = panda_status + else: + pass + # content['content_metadata']['panda_id'] = panda_id + content['substatus'] = panda_status + + update_contents.append(content) + num_updated_contents += 1 + else: + num_unupdated_contents += 1 + self.logger.debug("get_update_contents, num_updated_contents: %s, num_unupdated_contents: %s" % (num_updated_contents, num_unupdated_contents)) + return update_contents + + def poll_panda_task(self, processing=None, input_output_maps=None): + task_id = None + try: + from pandatools import Client + + if processing: + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + if task_id is None: + task_id = self.get_panda_task_id(processing) + + if task_id: + # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + self.logger.debug("poll_panda_task, task_id: %s" % str(task_id)) + task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) + self.logger.debug("poll_panda_task, task_info[0]: %s" % str(task_info[0])) + if task_info[0] != 0: + self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) + return ProcessingStatus.Submitting, {} + + task_info = task_info[1] + + processing_status = self.get_processing_status_from_panda_status(task_info["status"]) + + if processing_status in [ProcessingStatus.SubFinished]: + if self.retry_number < self.num_retries: + self.reactivate_processing(processing) + processing_status = ProcessingStatus.Submitted + self.retry_number += 1 + + all_jobs_ids = task_info['PandaID'] + + terminated_jobs, inputname_mapid_map = self.get_job_maps(input_output_maps) + self.logger.debug("poll_panda_task, task_id: %s, all jobs: %s, terminated_jobs: %s" % (str(task_id), len(all_jobs_ids), len(terminated_jobs))) + + all_jobs_ids = set(all_jobs_ids) + terminated_jobs = set(terminated_jobs) + unterminated_jobs = all_jobs_ids - terminated_jobs + + inputname_jobid_map = self.poll_panda_jobs(unterminated_jobs) + intersection_keys = set(inputname_mapid_map.keys()) & set(inputname_jobid_map.keys()) + + updated_contents = self.get_update_contents(list(intersection_keys), inputname_mapid_map, inputname_jobid_map) + + final_update_contents = [] + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if updated_contents: + # there are still polling contents, should not terminate the task. + log_warn = "Processing (%s) with panda task id (%s) is %s, however there are still updated_contents[:5]: %s" % (processing['processing_id'], + task_id, + processing_status, + str(updated_contents[:5])) + log_warn = log_warn + ". Keep the processing status as running now." + self.logger.warn(log_warn) + processing_status = ProcessingStatus.Running + elif list(unterminated_jobs): + log_warn = "Processing (%s) with panda task id (%s) is %s, however there are still unterminated_jobs[:5]: %s" % (processing['processing_id'], + task_id, + processing_status, + str(list(unterminated_jobs)[:5])) + log_warn = log_warn + ". Keep the processing status as running now." + self.logger.warn(log_warn) + processing_status = ProcessingStatus.Running + else: + # unsubmitted_inputnames = set(inputname_mapid_map.keys()) - set(inputname_jobid_map.keys()) + # unsubmitted_inputnames = list(unsubmitted_inputnames) + # if unsubmitted_inputnames: + # log_warn = "Processing (%s) with panda task id (%s) is %s, however there are still unsubmitted_inputnames[:5]: %s" % (processing['processing_id'], + # task_id, + # processing_status, + # str(unsubmitted_inputnames[:5])) + # log_warn = log_warn + ". Keep the processing status as running now." + # self.logger.warn(log_warn) + # processing_status = ProcessingStatus.Running + + for inputname in inputname_mapid_map: + map_id_contents = inputname_mapid_map[inputname] + contents = map_id_contents['outputs'] + for content in contents: + if (content['substatus'] not in [ContentStatus.Available, ContentStatus.FakeAvailable, ContentStatus.FinalFailed]): + content['content_metadata']['old_final_status'] = content['substatus'] + content['substatus'] = ContentStatus.FinalFailed + # final_update_contents.append(content) # TODO: mark other contents to Missing + + if final_update_contents: + processing_status = ProcessingStatus.Running + return processing_status, updated_contents + final_update_contents + else: + return ProcessingStatus.Failed, [] + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + self.logger.error(msg) + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException(msg) + return ProcessingStatus.Submitting, [] + def kill_processing(self, processing): try: if processing: @@ -799,7 +996,7 @@ def poll_processing_updates(self, processing, input_output_maps): update_processing = {} reset_expired_at = False reactive_contents = [] - # self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) + self.logger.debug("poll_processing_updates, input_output_maps.keys[:5]: %s" % str(list(input_output_maps.keys())[:5])) if processing: proc = processing['processing_metadata']['processing'] @@ -841,7 +1038,7 @@ def poll_processing_updates(self, processing, input_output_maps): processing_status, poll_updated_contents = self.poll_panda_task(processing=processing, input_output_maps=input_output_maps) self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) - # self.logger.debug("poll_processing_updates, update_contents[:100]: %s" % str(poll_updated_contents[:100])) + self.logger.debug("poll_processing_updates, update_contents[:10]: %s" % str(poll_updated_contents[:10])) if poll_updated_contents: proc.has_new_updates() From a069e36b185a8d71336203799185b4243771656c Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 2 Dec 2021 16:28:10 +0100 Subject: [PATCH 147/156] new atlas local panda --- .../atlas/workflowv2/atlaslocalpandawork.py | 433 ++++++++++++++++++ 1 file changed, 433 insertions(+) create mode 100644 atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py diff --git a/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py new file mode 100644 index 00000000..be081c37 --- /dev/null +++ b/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2020 - 2021 + + +import json +import os +import traceback + +# from rucio.client.client import Client as RucioClient +from rucio.common.exception import (CannotAuthenticate as RucioCannotAuthenticate) + +from idds.common import exceptions +from idds.common.constants import (ContentStatus, + ProcessingStatus, WorkStatus) +from idds.common.utils import extract_scope_atlas +# from idds.workflowv2.work import Work, Processing +# from idds.workflowv2.workflow import Condition +from .atlaspandawork import ATLASPandaWork + + +class ATLASLocalPandaWork(ATLASPandaWork): + def __init__(self, task_parameters=None, + work_tag='atlas', exec_type='panda', work_id=None, + primary_input_collection=None, other_input_collections=None, + input_collections=None, + primary_output_collection=None, other_output_collections=None, + output_collections=None, log_collections=None, + logger=None, + num_retries=5, + ): + + super(ATLASLocalPandaWork, self).__init__(task_parameters=task_parameters, + work_tag=work_tag, + exec_type=exec_type, + work_id=work_id, + primary_input_collection=primary_input_collection, + other_input_collections=other_input_collections, + input_collections=input_collections, + primary_output_collection=primary_output_collection, + other_output_collections=other_output_collections, + output_collections=output_collections, + log_collections=log_collections, + logger=logger, + num_retries=num_retries) + self.work_dir = "/tmp" + self.output_files = [] + + def set_agent_attributes(self, attrs, req_attributes=None): + if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: + attrs['life_time'] = None + super(ATLASLocalPandaWork, self).set_agent_attributes(attrs) + if self.agent_attributes and 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: + self.num_retries = int(self.agent_attributes['num_retries']) + if self.agent_attributes and 'work_dir' in self.agent_attributes and self.agent_attributes['work_dir']: + self.work_dir = int(self.agent_attributes['work_dir']) + + def parse_task_parameters(self, task_parameters): + super(ATLASLocalPandaWork, self).parse_task_parameters(task_parameters) + + try: + + if 'jobParameters' in self.task_parameters: + jobParameters = self.task_parameters['jobParameters'] + for jobP in jobParameters: + if type(jobP) in [dict]: + if 'dataset' in jobP and 'param_type' in jobP: + if jobP['param_type'] == 'output' and 'value' in jobP: + output_c = jobP['dataset'] + scope, name = extract_scope_atlas(output_c, scopes=[]) + # output_coll = {'scope': scope, 'name': name} + output_f = jobP['value'] + output_file = {'scope': scope, 'name': name, 'file': output_f} + self.output_files.append(output_file) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + self.add_errors(str(ex)) + + def renew_parameters_from_attributes(self): + if not self.task_parameters: + return + + try: + if 'taskName' in self.task_parameters: + self.task_name = self.task_parameters['taskName'] + self.task_name = self.renew_parameter(self.task_name) + self.set_work_name(self.task_name) + + if 'prodSourceLabel' in self.task_parameters: + self.task_type = self.task_parameters['prodSourceLabel'] + + if 'jobParameters' in self.task_parameters: + jobParameters = self.task_parameters['jobParameters'] + for jobP in jobParameters: + if type(jobP) in [dict]: + for key in jobP: + if jobP[key] and type(jobP[key]) in [str]: + jobP[key] = self.renew_parameter(jobP[key]) + for coll_id in self.collections: + coll_name = self.collections[coll_id].name + self.collections[coll_id].name = self.renew_parameter(coll_name) + + output_files = self.output_files + self.output_files = [] + for output_file in output_files: + output_file['name'] = self.renew_parameter(output_file['name']) + output_file['file'] = self.renew_parameter(output_file['file']) + self.output_files.append(output_file) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + self.add_errors(str(ex)) + + def set_output_data(self, data): + self.output_data = data + if type(data) in [dict]: + for key in data: + new_key = "user_" + str(key) + setattr(self, new_key, data[key]) + + def ping_output_files(self): + try: + rucio_client = self.get_rucio_client() + for output_i in range(len(self.output_files)): + output_file = self.output_files[output_i] + f_parts = output_file['file'].split(".") + pos = output_file['file'].find("$") + begin_part = output_file['file'][:pos] + i = 0 + for i in range(len(f_parts) - 1, -1, -1): + if "$" in f_parts[i]: + break + end_parts = f_parts[i + 1:] + end_part = '.'.join(end_parts) + + files = rucio_client.list_files(scope=output_file['scope'], + name=output_file['name']) + for f in files: + if ((not begin_part) or f['name'].startswith(begin_part)) and ((not end_part) or f['name'].endswith(end_part)): + self.output_files[output_i]['lfn'] = f['name'] + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def download_output_file_rucio(self, file_items): + try: + client = self.get_rucio_download_client() + outputs = client.download_dids(file_items) + ret = {} + for output in outputs: + if 'dest_file_paths' in output and output['dest_file_paths']: + ret[output['name']] = output['dest_file_paths'][0] + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def download_output_files(self, processing): + try: + file_items = [] + for output_i in range(len(self.output_files)): + if 'lfn' in self.output_files[output_i]: + self.logger.debug("download_output_files, Processing (%s) lfn for %s: %s" % (processing['processing_id'], + self.output_files[output_i]['file'], + self.output_files[output_i]['lfn'])) + file_items.append("%s:%s" % (self.output_files[output_i]['scope'], self.output_files[output_i]['lfn'])) + else: + self.logger.warn("download_output_files, Processing (%s) lfn for %s not found" % (processing['processing_id'], + self.output_files[output_i]['file'])) + + pfn_items = self.download_output_files_rucio(file_items) + for output_i in range(len(self.output_files)): + if 'lfn' in self.output_files[output_i]: + pfn = pfn_items[self.output_files[output_i]['lfn']] + self.output_files[output_i]['pfn'] = pfn + self.logger.info("download_output_files, Processing (%s) pfn for %s: %s" % (processing['processing_id'], + self.output_files[output_i]['file'], + self.output_files[output_i]['pfn'])) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def parse_output_file(self, pfn): + try: + if not os.path.exists(pfn): + self.logger.warn("%s doesn't exist" % pfn) + return {} + else: + with open(pfn, 'r') as f: + data = f.read() + outputs = json.loads(data) + return outputs + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return {} + + def parse_output_files(self, processing): + try: + output_data = {} + for output_i in range(len(self.output_files)): + if 'pfn' in self.output_files[output_i]: + data = self.parse_output_file(self.output_files[output_i]['pfn']) + if type(data) in [dict]: + output_data.update(data) + except Exception as ex: + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + + def process_outputs(self, processing): + self.download_output_files(processing) + output_data = self.parse_output_files(processing) + return output_data + + def get_rucio_download_client(self): + try: + from rucio.client.downloadclient import DownloadClient + client = DownloadClient() + except RucioCannotAuthenticate as error: + self.logger.error(error) + self.logger.error(traceback.format_exc()) + raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc())) + return client + + def poll_panda_task_output(self, processing=None, input_output_maps=None): + task_id = None + try: + from pandatools import Client + + if processing: + output_metadata = {} + proc = processing['processing_metadata']['processing'] + task_id = proc.workload_id + if task_id is None: + task_id = self.get_panda_task_id(processing) + + if task_id: + # ret_ids = Client.getPandaIDsWithTaskID(task_id, verbose=False) + task_info = Client.getJediTaskDetails({'jediTaskID': task_id}, True, True, verbose=False) + self.logger.info("poll_panda_task, task_info: %s" % str(task_info)) + if task_info[0] != 0: + self.logger.warn("poll_panda_task %s, error getting task status, task_info: %s" % (task_id, str(task_info))) + return ProcessingStatus.Submitting, [], {} + + task_info = task_info[1] + + processing_status = self.get_processing_status_from_panda_status(task_info["status"]) + + if processing_status in [ProcessingStatus.SubFinished]: + if self.retry_number < self.num_retries: + self.reactivate_processing(processing) + processing_status = ProcessingStatus.Submitted + self.retry_number += 1 + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished]: + output_metdata = self.process_outputs(processing) + + return processing_status, [], {}, output_metdata + else: + return ProcessingStatus.Failed, [], {}, output_metadata + except Exception as ex: + msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) + self.logger.error(msg) + self.logger.error(ex) + self.logger.error(traceback.format_exc()) + # raise exceptions.IDDSException(msg) + return ProcessingStatus.Submitting, [], {}, {} + + def poll_processing_updates(self, processing, input_output_maps): + """ + *** Function called by Carrier agent. + """ + updated_contents = [] + update_processing = {} + reset_expired_at = False + reactive_contents = [] + # self.logger.debug("poll_processing_updates, input_output_maps: %s" % str(input_output_maps)) + + if processing: + proc = processing['processing_metadata']['processing'] + if proc.tocancel: + self.logger.info("Cancelling processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tocancel = False + proc.polling_retries = 0 + elif proc.tosuspend: + self.logger.info("Suspending processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing_force(processing) + # self.kill_processing(processing) + proc.tosuspend = False + proc.polling_retries = 0 + elif proc.toresume: + self.logger.info("Resuming processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.reactivate_processing(processing) + reset_expired_at = True + proc.toresume = False + proc.polling_retries = 0 + proc.has_new_updates() + # reactive_contents = self.reactive_contents(input_output_maps) + # elif self.is_processing_expired(processing): + elif proc.toexpire: + self.logger.info("Expiring processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.toexpire = False + proc.polling_retries = 0 + elif proc.tofinish or proc.toforcefinish: + self.logger.info("Finishing processing (processing id: %s, jediTaskId: %s)" % (processing['processing_id'], proc.workload_id)) + self.kill_processing(processing) + proc.tofinish = False + proc.toforcefinish = False + proc.polling_retries = 0 + + processing_status, poll_updated_contents, new_input_output_maps, output_metadata = self.poll_panda_task_output(processing=processing, input_output_maps=input_output_maps) + self.logger.debug("poll_processing_updates, processing_status: %s" % str(processing_status)) + self.logger.debug("poll_processing_updates, update_contents: %s" % str(poll_updated_contents)) + self.logger.debug("poll_processing_updates, output_metadata: %s" % str(output_metadata)) + + if poll_updated_contents: + proc.has_new_updates() + for content in poll_updated_contents: + updated_content = {'content_id': content['content_id'], + 'substatus': content['substatus'], + 'content_metadata': content['content_metadata']} + updated_contents.append(updated_content) + + content_substatus = {'finished': 0, 'unfinished': 0} + for map_id in input_output_maps: + outputs = input_output_maps[map_id]['outputs'] + for content in outputs: + if content.get('substatus', ContentStatus.New) != ContentStatus.Available: + content_substatus['unfinished'] += 1 + else: + content_substatus['finished'] += 1 + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] and updated_contents: + self.logger.info("Processing %s is terminated, but there are still contents to be flushed. Waiting." % (proc.workload_id)) + # there are still polling contents, should not terminate the task. + processing_status = ProcessingStatus.Running + + if processing_status in [ProcessingStatus.SubFinished] and content_substatus['finished'] > 0 and content_substatus['unfinished'] == 0: + # found that a 'done' panda task has got a 'finished' status. Maybe in this case 'finished' is a transparent status. + if proc.polling_retries is None: + proc.polling_retries = 0 + + if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed]: + if proc.polling_retries is not None and proc.polling_retries < 3: + self.logger.info("processing %s polling_retries(%s) < 3, keep running" % (processing['processing_id'], proc.polling_retries)) + processing_status = ProcessingStatus.Running + proc.polling_retries += 1 + else: + proc.polling_retries = 0 + + if proc.in_operation_time(): + processing_status = ProcessingStatus.Running + + update_processing = {'processing_id': processing['processing_id'], + 'parameters': {'status': processing_status, + 'output_metadata': output_metadata}} + + if reset_expired_at: + processing['expired_at'] = None + update_processing['parameters']['expired_at'] = None + proc.polling_retries = 0 + # if (processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished, ProcessingStatus.Failed] + # or processing['status'] in [ProcessingStatus.Resuming]): # noqa W503 + # using polling_retries to poll it again when panda may update the status in a delay(when issuing retryTask, panda will not update it without any delay). + update_processing['parameters']['status'] = ProcessingStatus.Resuming + proc.status = update_processing['parameters']['status'] + + self.logger.debug("poll_processing_updates, task: %s, update_processing: %s" % + (proc.workload_id, str(update_processing))) + self.logger.debug("poll_processing_updates, task: %s, updated_contents: %s" % + (proc.workload_id, str(updated_contents))) + self.logger.debug("poll_processing_updates, task: %s, reactive_contents: %s" % + (proc.workload_id, str(reactive_contents))) + return update_processing, updated_contents + reactive_contents, new_input_output_maps + + def syn_work_status(self, registered_input_output_maps, all_updates_flushed=True, output_statistics={}, to_release_input_contents=[]): + super(ATLASLocalPandaWork, self).syn_work_status(registered_input_output_maps, all_updates_flushed, output_statistics, to_release_input_contents) + # self.get_status_statistics(registered_input_output_maps) + self.status_statistics = output_statistics + + self.logger.debug("syn_work_status, self.active_processings: %s" % str(self.active_processings)) + self.logger.debug("syn_work_status, self.has_new_inputs(): %s" % str(self.has_new_inputs)) + self.logger.debug("syn_work_status, coll_metadata_is_open: %s" % + str(self.collections[self._primary_input_collection].coll_metadata['is_open'])) + self.logger.debug("syn_work_status, primary_input_collection_status: %s" % + str(self.collections[self._primary_input_collection].status)) + + self.logger.debug("syn_work_status(%s): is_processings_terminated: %s" % (str(self.get_processing_ids()), str(self.is_processings_terminated()))) + self.logger.debug("syn_work_status(%s): is_input_collections_closed: %s" % (str(self.get_processing_ids()), str(self.is_input_collections_closed()))) + self.logger.debug("syn_work_status(%s): has_new_inputs: %s" % (str(self.get_processing_ids()), str(self.has_new_inputs))) + self.logger.debug("syn_work_status(%s): has_to_release_inputs: %s" % (str(self.get_processing_ids()), str(self.has_to_release_inputs()))) + self.logger.debug("syn_work_status(%s): to_release_input_contents: %s" % (str(self.get_processing_ids()), str(to_release_input_contents))) + + if self.is_processings_terminated() and self.is_input_collections_closed() and not self.has_new_inputs and not self.has_to_release_inputs() and not to_release_input_contents: + # if not self.is_all_outputs_flushed(registered_input_output_maps): + if not all_updates_flushed: + self.logger.warn("The work processings %s is terminated. but not all outputs are flushed. Wait to flush the outputs then finish the transform" % str(self.get_processing_ids())) + return + + if self.is_processings_finished(): + self.status = WorkStatus.Finished + elif self.is_processings_subfinished(): + self.status = WorkStatus.SubFinished + elif self.is_processings_failed(): + self.status = WorkStatus.Failed + elif self.is_processings_expired(): + self.status = WorkStatus.Expired + elif self.is_processings_cancelled(): + self.status = WorkStatus.Cancelled + elif self.is_processings_suspended(): + self.status = WorkStatus.Suspended + elif self.is_processings_running(): + self.status = WorkStatus.Running + else: + self.status = WorkStatus.Transforming + + if self.is_processings_terminated() or self.is_processings_running() or self.is_processings_started(): + self.started = True From e0735f92fff387511cba9f062c5e26df79f80452 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 2 Dec 2021 16:43:57 +0100 Subject: [PATCH 148/156] set download dir --- .../idds/atlas/workflowv2/atlaslocalpandawork.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py index be081c37..b1136a48 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py @@ -165,6 +165,15 @@ def download_output_file_rucio(self, file_items): self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + def get_download_dir(self, processing): + req_dir = 'request_%s_%s/transform_%s' % (processing['request_id'], + processing['workload_id'], + processing['transform_id']) + d_dir = os.path.join(self.work_dir, req_dir) + if not os.path.exists(d_dir): + os.makedirs(d_dir) + return d_dir + def download_output_files(self, processing): try: file_items = [] @@ -173,7 +182,10 @@ def download_output_files(self, processing): self.logger.debug("download_output_files, Processing (%s) lfn for %s: %s" % (processing['processing_id'], self.output_files[output_i]['file'], self.output_files[output_i]['lfn'])) - file_items.append("%s:%s" % (self.output_files[output_i]['scope'], self.output_files[output_i]['lfn'])) + file_item = {'did': "%s:%s" % (self.output_files[output_i]['scope'], self.output_files[output_i]['lfn']), + 'base_dir': self.get_download_dir(processing), + 'no_subdir': True} + file_items.append(file_item) else: self.logger.warn("download_output_files, Processing (%s) lfn for %s not found" % (processing['processing_id'], self.output_files[output_i]['file'])) From 34e00833883edfb4da8848986b30707768a6d36e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Thu, 2 Dec 2021 16:46:20 +0100 Subject: [PATCH 149/156] new version 0.9.0 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 3a62034a..e3a1b9c6 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index f82788bb..638fe0cc 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.8.6 - - idds-workflow==0.8.6 \ No newline at end of file + - idds-common==0.9.0 + - idds-workflow==0.9.0 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 3a62034a..e3a1b9c6 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 2d3ebf8d..48d00e77 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.8.6 - - idds-workflow==0.8.6 \ No newline at end of file + - idds-common==0.9.0 + - idds-workflow==0.9.0 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 3a62034a..e3a1b9c6 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 64b532d2..c802f308 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 23885dde..ba324cd8 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.8.6 - - idds-workflow==0.8.6 \ No newline at end of file + - idds-common==0.9.0 + - idds-workflow==0.9.0 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 3a62034a..e3a1b9c6 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 3813dcf8..5bf8cfb5 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.8.6 - - idds-workflow==0.8.6 - - idds-client==0.8.6 \ No newline at end of file + - idds-common==0.9.0 + - idds-workflow==0.9.0 + - idds-client==0.9.0 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index 3a62034a..e3a1b9c6 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/website/version.py b/website/version.py index 3a62034a..e3a1b9c6 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 3a62034a..e3a1b9c6 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.8.6" +release_version = "0.9.0" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index c1966408..4d9a5344 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.8.6 \ No newline at end of file + - idds-common==0.9.0 \ No newline at end of file From becc5140389c92eecc521a91a62db492a45a8dc1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:27:33 +0100 Subject: [PATCH 150/156] fix download and parse output files --- .../atlas/workflowv2/atlaslocalpandawork.py | 130 ++++++++++++------ 1 file changed, 90 insertions(+), 40 deletions(-) diff --git a/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py index b1136a48..6bd4e01c 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py @@ -11,6 +11,7 @@ import json import os +import re import traceback # from rucio.client.client import Client as RucioClient @@ -36,6 +37,9 @@ def __init__(self, task_parameters=None, num_retries=5, ): + self.work_dir = "/tmp" + self.output_files = [] + super(ATLASLocalPandaWork, self).__init__(task_parameters=task_parameters, work_tag=work_tag, exec_type=exec_type, @@ -49,8 +53,6 @@ def __init__(self, task_parameters=None, log_collections=log_collections, logger=logger, num_retries=num_retries) - self.work_dir = "/tmp" - self.output_files = [] def set_agent_attributes(self, attrs, req_attributes=None): if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: @@ -59,14 +61,13 @@ def set_agent_attributes(self, attrs, req_attributes=None): if self.agent_attributes and 'num_retries' in self.agent_attributes and self.agent_attributes['num_retries']: self.num_retries = int(self.agent_attributes['num_retries']) if self.agent_attributes and 'work_dir' in self.agent_attributes and self.agent_attributes['work_dir']: - self.work_dir = int(self.agent_attributes['work_dir']) + self.work_dir = self.agent_attributes['work_dir'] def parse_task_parameters(self, task_parameters): super(ATLASLocalPandaWork, self).parse_task_parameters(task_parameters) try: - - if 'jobParameters' in self.task_parameters: + if self.task_parameters and 'jobParameters' in self.task_parameters: jobParameters = self.task_parameters['jobParameters'] for jobP in jobParameters: if type(jobP) in [dict]: @@ -122,48 +123,73 @@ def renew_parameters_from_attributes(self): def set_output_data(self, data): self.output_data = data - if type(data) in [dict]: + if data and type(data) in [dict]: for key in data: new_key = "user_" + str(key) setattr(self, new_key, data[key]) + def match_pattern_file(self, pattern, lfn): + pattern1 = "\\$[_a-zA-Z0-9]+" + pattern2 = "\\$\\{[_a-zA-Z0-9\\/]+\\}" + while True: + m = re.search(pattern1, pattern) + if m: + pattern = pattern.replace(m.group(0), "*") + else: + break + while True: + m = re.search(pattern2, pattern) + if m: + pattern = pattern.replace(m.group(0), "*") + else: + break + + pattern = pattern.replace(".", "\\.") + pattern = pattern.replace("*", ".*") + pattern = pattern + "$" + + m = re.search(pattern, lfn) + if m: + return True + return False + def ping_output_files(self): try: rucio_client = self.get_rucio_client() for output_i in range(len(self.output_files)): output_file = self.output_files[output_i] - f_parts = output_file['file'].split(".") - pos = output_file['file'].find("$") - begin_part = output_file['file'][:pos] - i = 0 - for i in range(len(f_parts) - 1, -1, -1): - if "$" in f_parts[i]: - break - end_parts = f_parts[i + 1:] - end_part = '.'.join(end_parts) - files = rucio_client.list_files(scope=output_file['scope'], name=output_file['name']) + files = [f for f in files] + self.logger.debug("ping_output_files found files for dataset(%s:%s): %s" % (output_file['scope'], + output_file['name'], + str([f['name'] for f in files]))) for f in files: - if ((not begin_part) or f['name'].startswith(begin_part)) and ((not end_part) or f['name'].endswith(end_part)): + if self.match_pattern_file(output_file['file'], f['name']): self.output_files[output_i]['lfn'] = f['name'] + return True except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return False - def download_output_file_rucio(self, file_items): + def download_output_files_rucio(self, file_items): try: - client = self.get_rucio_download_client() - outputs = client.download_dids(file_items) ret = {} - for output in outputs: - if 'dest_file_paths' in output and output['dest_file_paths']: - ret[output['name']] = output['dest_file_paths'][0] + self.logger.debug("download_output_files_rucio: %s" % str(file_items)) + if file_items: + client = self.get_rucio_download_client() + outputs = client.download_dids(file_items) + for output in outputs: + if 'dest_file_paths' in output and output['dest_file_paths']: + ret[output['name']] = output['dest_file_paths'][0] + return ret except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return {} def get_download_dir(self, processing): req_dir = 'request_%s_%s/transform_%s' % (processing['request_id'], @@ -176,6 +202,7 @@ def get_download_dir(self, processing): def download_output_files(self, processing): try: + failed_items = [] file_items = [] for output_i in range(len(self.output_files)): if 'lfn' in self.output_files[output_i]: @@ -193,15 +220,22 @@ def download_output_files(self, processing): pfn_items = self.download_output_files_rucio(file_items) for output_i in range(len(self.output_files)): if 'lfn' in self.output_files[output_i]: - pfn = pfn_items[self.output_files[output_i]['lfn']] - self.output_files[output_i]['pfn'] = pfn - self.logger.info("download_output_files, Processing (%s) pfn for %s: %s" % (processing['processing_id'], - self.output_files[output_i]['file'], - self.output_files[output_i]['pfn'])) + if self.output_files[output_i]['lfn'] in pfn_items: + pfn = pfn_items[self.output_files[output_i]['lfn']] + self.output_files[output_i]['pfn'] = pfn + self.logger.info("download_output_files, Processing (%s) pfn for %s: %s" % (processing['processing_id'], + self.output_files[output_i]['file'], + self.output_files[output_i]['pfn'])) + else: + self.logger.info("download_output_files, Processing (%s) pfn cannot be found for %s" % (processing['processing_id'], + self.output_files[output_i]['file'])) + failed_items.append(self.output_files[output_i]) + return failed_items except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return [] def parse_output_file(self, pfn): try: @@ -214,8 +248,8 @@ def parse_output_file(self, pfn): outputs = json.loads(data) return outputs except Exception as ex: - self.logger.error(ex) - self.logger.error(traceback.format_exc()) + # self.logger.error(ex) + # self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) return {} @@ -227,15 +261,26 @@ def parse_output_files(self, processing): data = self.parse_output_file(self.output_files[output_i]['pfn']) if type(data) in [dict]: output_data.update(data) + return True, output_data except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) - raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + # raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) + return False, output_data def process_outputs(self, processing): - self.download_output_files(processing) - output_data = self.parse_output_files(processing) - return output_data + ping_status = self.ping_output_files() + self.logger.debug("ping_output_files(Processing_id: %s), status: %s" % (processing['processing_id'], ping_status)) + + failed_items = self.download_output_files(processing) + self.logger.debug("download_output_files(Processing_id: %s), failed_items: %s" % (processing['processing_id'], str(failed_items))) + + parse_status, output_data = self.parse_output_files(processing) + self.logger.debug("parse_output_files(Processing_id: %s), parse_status: %s, output_data: %s" % (processing['processing_id'], parse_status, str(output_data))) + + if ping_status and not failed_items and parse_status: + return True, output_data + return False, output_data def get_rucio_download_client(self): try: @@ -277,9 +322,14 @@ def poll_panda_task_output(self, processing=None, input_output_maps=None): processing_status = ProcessingStatus.Submitted self.retry_number += 1 if processing_status in [ProcessingStatus.SubFinished, ProcessingStatus.Finished]: - output_metdata = self.process_outputs(processing) - - return processing_status, [], {}, output_metdata + output_status, output_metadata = self.process_outputs(processing) + if not output_status: + err = "Failed to process processing(processing_id: %s, task_id: %s) outputs" % (processing['processing_id'], task_id) + self.logger.error(err) + self.add_errors(err) + processing_status = ProcessingStatus.Failed + + return processing_status, [], {}, output_metadata else: return ProcessingStatus.Failed, [], {}, output_metadata except Exception as ex: From 3bd8b3dcc32003f78c8dd2eb1b0adf2a2c9be2f8 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:28:16 +0100 Subject: [PATCH 151/156] fix propagate output info to global parameters --- workflow/lib/idds/workflowv2/work.py | 29 ++++++++++++++++++++++-- workflow/lib/idds/workflowv2/workflow.py | 9 +++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/workflow/lib/idds/workflowv2/work.py b/workflow/lib/idds/workflowv2/work.py index 4557859f..b4f7937f 100644 --- a/workflow/lib/idds/workflowv2/work.py +++ b/workflow/lib/idds/workflowv2/work.py @@ -519,7 +519,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.suspended_processings = [] self.old_processings = [] self.terminated_msg = "" - self.output_data = None + self.output_data = {} self.parameters_for_next_task = None self.status_statistics = {} @@ -628,6 +628,18 @@ def parameters(self): def parameters(self, value): self.add_metadata_item('parameters', value) + @property + def output_data(self): + return self.get_metadata_item('output_data', {}) + + @output_data.setter + def output_data(self, value): + self.add_metadata_item('output_data', value) + if value and type(value) in [dict]: + for key in value: + new_key = "user_" + str(key) + setattr(self, new_key, value[key]) + @property def work_id(self): return self.get_metadata_item('work_id', None) @@ -1014,6 +1026,18 @@ def sync_global_parameters(self, global_parameters): for key in global_parameters: setattr(self, key, global_parameters[key]) + def get_global_parameter_from_output_data(self, key): + self.logger.debug("get_global_parameter_from_output_data, key: %s, output_data: %s" % (key, str(self.output_data))) + gp_output_data = {} + if self.output_data and type(self.output_data) in [dict]: + for key in self.output_data: + new_key = "user_" + str(key) + gp_output_data[new_key] = self.output_data[key] + if key in gp_output_data: + return True, gp_output_data[key] + else: + return False, None + def renew_parameters_from_attributes(self): pass @@ -1066,7 +1090,7 @@ def clean_work(self): self.suspended_processings = [] self.old_processings = [] self.terminated_msg = "" - self.output_data = None + self.output_data = {} self.parameters_for_next_task = None def set_agent_attributes(self, attrs, req_attributes=None): @@ -1964,6 +1988,7 @@ def sync_work_data(self, status, substatus, work): self.status_statistics = work.status_statistics self.processings = work.processings + self.output_data = work.output_data """ self.status = WorkStatus(status.value) diff --git a/workflow/lib/idds/workflowv2/workflow.py b/workflow/lib/idds/workflowv2/workflow.py index 4ffd17d7..915d87c4 100644 --- a/workflow/lib/idds/workflowv2/workflow.py +++ b/workflow/lib/idds/workflowv2/workflow.py @@ -846,10 +846,16 @@ def set_global_parameters(self, value): self.global_parameters = value def sync_global_parameters_from_work(self, work): + self.log_debug("work %s is_terminated, global_parameters: %s" % (work.get_internal_id(), str(self.global_parameters))) if self.global_parameters: for key in self.global_parameters: - if hasattr(work, key): + status, value = work.get_global_parameter_from_output_data(key) + self.log_debug("work %s get_global_parameter_from_output_data(key: %s) results(%s:%s)" % (work.get_internal_id(), key, status, value)) + if status: + self.global_parameters[key] = value + elif hasattr(work, key): self.global_parameters[key] = getattr(work, key) + self.set_global_parameters(self.global_parameters) @property def loop_condition(self): @@ -1452,6 +1458,7 @@ def sync_works(self): work.sync_works() if work.is_terminated(): + self.log_debug("work %s is_terminated, sync_global_parameters_from_work" % (work.get_internal_id())) self.set_source_parameters(work.get_internal_id()) self.sync_global_parameters_from_work(work) From 829190fbc43abac10eb8743101c60e305f56219d Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:28:59 +0100 Subject: [PATCH 152/156] fix workflow sync --- main/lib/idds/agents/clerk/clerk.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/main/lib/idds/agents/clerk/clerk.py b/main/lib/idds/agents/clerk/clerk.py index 566717c1..9930872c 100644 --- a/main/lib/idds/agents/clerk/clerk.py +++ b/main/lib/idds/agents/clerk/clerk.py @@ -305,18 +305,6 @@ def process_running_request_real(self, req): self.logger.info("process_running_request: request_id: %s" % req['request_id']) wf = req['request_metadata']['workflow'] - new_transforms = [] - if req['status'] in [RequestStatus.Transforming]: - # new works - works = wf.get_new_works() - for work in works: - # new_work = work.copy() - new_work = work - new_work.add_proxy(wf.get_proxy()) - new_transform = self.generate_transform(req, new_work) - new_transforms.append(new_transform) - self.logger.debug("Processing request(%s): new transforms: %s" % (req['request_id'], str(new_transforms))) - # current works works = wf.get_all_works() # print(works) @@ -330,6 +318,18 @@ def process_running_request_real(self, req): work.sync_work_data(status=tf['status'], substatus=tf['substatus'], work=transform_work) wf.refresh_works() + new_transforms = [] + if req['status'] in [RequestStatus.Transforming]: + # new works + works = wf.get_new_works() + for work in works: + # new_work = work.copy() + new_work = work + new_work.add_proxy(wf.get_proxy()) + new_transform = self.generate_transform(req, new_work) + new_transforms.append(new_transform) + self.logger.debug("Processing request(%s): new transforms: %s" % (req['request_id'], str(new_transforms))) + is_operation = False if wf.is_terminated(): if wf.is_finished(): From 91b65780b69dffe7810166daae6cc893da2d7310 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:29:46 +0100 Subject: [PATCH 153/156] fix env for python 3.9.7 --- main/tools/env/environment.yml | 4 ++-- main/tools/env/install_idds_full.sh | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 5bf8cfb5..adbc8bbb 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -1,6 +1,6 @@ name: iDDS dependencies: -- python==3.6.2 +- python==3.9.7 - pip - pip: - requests # requests @@ -24,4 +24,4 @@ dependencies: - nevergrad # nevergrad hyper parameter optimization - idds-common==0.9.0 - idds-workflow==0.9.0 - - idds-client==0.9.0 \ No newline at end of file + - idds-client==0.9.0 diff --git a/main/tools/env/install_idds_full.sh b/main/tools/env/install_idds_full.sh index 35c10cdc..43eca3af 100644 --- a/main/tools/env/install_idds_full.sh +++ b/main/tools/env/install_idds_full.sh @@ -2,6 +2,10 @@ # as root yum install -y httpd.x86_64 conda gridsite mod_ssl.x86_64 httpd-devel.x86_64 gcc.x86_64 supervisor.noarch +# yum install -y gfal2-plugin-gridftp gfal2-plugin-file.x86_64 gfal2-plugin-http.x86_64 gfal2-plugin-xrootd.x86_64 gfal2-python.x86_64 gfal2-python3.x86_64 gfal2-all.x86_64 +conda install -c conda-forge python-gfal2 +pip install requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad + mkdir /opt/idds mkdir /opt/idds_source mkdir /opt/idds From 4c95eb8033914582e582e0308cafc4faa82e3c61 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:30:35 +0100 Subject: [PATCH 154/156] new rest cfg for py39 --- .../httpd-idds-443-py39-cc7.conf.template | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 main/etc/idds/rest/httpd-idds-443-py39-cc7.conf.template diff --git a/main/etc/idds/rest/httpd-idds-443-py39-cc7.conf.template b/main/etc/idds/rest/httpd-idds-443-py39-cc7.conf.template new file mode 100644 index 00000000..4214f207 --- /dev/null +++ b/main/etc/idds/rest/httpd-idds-443-py39-cc7.conf.template @@ -0,0 +1,120 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Wen Guan, , 2019 + +# TimeOut 600 +# KeepAliveTimeout 600 + +# Built-in modules +LoadModule ssl_module /usr/lib64/httpd/modules/mod_ssl.so + +# # LoadModule log_config_module /usr/lib64/httpd/modules/mod_log_config.so +# # LoadModule ssl_module /usr/lib64/httpd/modules/mod_ssl.so +# # LoadModule gridsite_module /usr/lib64/httpd/modules/mod_gridsite.so +# # LoadModule mime_module /usr/lib64/httpd/modules/mod_mime.so +# # LoadModule dir_module /usr/lib64/httpd/modules/mod_dir.so +# # LoadModule alias_module /usr/lib64/httpd/modules/mod_alias.so +# # LoadModule cgi_module /usr/lib64/httpd/modules/mod_cgi.so + +# External modules +LoadModule gridsite_module /usr/lib64/httpd/modules/mod_gridsite.so +#LoadModule wsgi_module /usr/lib64/httpd/modules/mod_wsgi.so +LoadModule wsgi_module {python_site_packages_path}/mod_wsgi/server/mod_wsgi-py36.cpython-36m-x86_64-linux-gnu.so +LoadModule wsgi_module {python_site_packages_path}/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so + +WSGIPythonHome {python_site_home_path} +WSGIPythonPath {python_site_packages_path} + + + WSGIDaemonProcess idds_daemon processes=25 threads=2 request-timeout=600 queue-timeout=600 python-home={python_site_home_path} python-path={python_site_packages_path} + WSGIProcessGroup idds_daemon + WSGIApplicationGroup %{GLOBAL} + WSGIScriptAlias /idds {python_site_bin_path}/idds.wsgi + # WSGIScriptAliasMatch ^/idds/(.+)$ /opt/idds/etc/idds/rest/test.wsgi + WSGISocketPrefix /var/log/idds/wsgisocks/wsgi + WSGIPassAuthorization On + + +Listen 443 + +RewriteEngine on +RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK) +RewriteRule .* - [F] +RedirectMatch 403 /\..*$ +TraceEnable off + +Alias "/website" "/opt/idds/website" +Alias "/monitor" "/opt/idds/monitor" + + + # ServerName aipanda182.cern.ch:443 + ServerAdmin wguan@cern.ch + + SSLEngine on + SSLCertificateFile /etc/grid-security/hostcert.pem + SSLCertificateKeyFile /etc/grid-security/hostkey.pem + SSLCACertificatePath /etc/grid-security/certificates + SSLCARevocationPath /etc/grid-security/certificates + SSLVerifyClient optional + SSLVerifyDepth 16 + SSLOptions +StdEnvVars +ExportCertData + + # CERN security recommendation to only allow the seven strongest ssl ciphers + SSLProtocol all -SSLv2 -SSLv3 + SSLCipherSuite HIGH:!CAMELLIA:!ADH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!3DES + SSLHonorCipherOrder on + + LogLevel debug + ErrorLog /var/log/idds/httpd_error_log + TransferLog /var/log/idds/httpd_access_log + + # Proxy authentication via mod_gridsite + + GridSiteIndexes on + GridSiteAuth on + GridSiteDNlists /etc/grid-security/dn-lists/ + GridSiteGSIProxyLimit 16 + GridSiteEnvs on + GridSiteACLPath /opt/idds/etc/idds/rest/gacl + + + + GridSiteIndexes on + GridSiteAuth on + GridSiteDNlists /etc/grid-security/dn-lists/ + GridSiteGSIProxyLimit 16 + GridSiteEnvs on + GridSiteACLPath /opt/idds/etc/idds/rest/gacl + # GridSiteMethods GET + + + + Order deny,allow + Allow from all + Require all granted + + + + Order deny,allow + Allow from all + Require all granted + + + + Order deny,allow + Allow from all + Require all granted + + + + Order deny,allow + Allow from all + Require all granted + DirectoryIndex dashboard.html + DirectoryIndex index.html + + From 22ab22cfb115a7b31a22058b4e85230ea152ade8 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:30:56 +0100 Subject: [PATCH 155/156] update tests --- common/lib/idds/common/dict_class.py | 2 ++ main/lib/idds/tests/core_tests.py | 12 ++++++------ main/lib/idds/tests/test_migrate_requests.py | 4 +++- monitor/conf.js | 12 ++++++------ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/common/lib/idds/common/dict_class.py b/common/lib/idds/common/dict_class.py index bdf1612f..6771e414 100644 --- a/common/lib/idds/common/dict_class.py +++ b/common/lib/idds/common/dict_class.py @@ -104,6 +104,8 @@ def from_dict(d): for key, value in d['attributes'].items(): if key == 'logger': continue + # elif key == 'output_data': + # continue else: value = DictClass.from_dict(value) setattr(impl, key, value) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 58f0bccc..8dd444aa 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -102,15 +102,15 @@ def show_works(req): print(work_ids) -reqs = get_requests(request_id=538, with_detail=True, with_metadata=True) +reqs = get_requests(request_id=229, with_detail=True, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) - print(json_dumps(req, sort_keys=True, indent=4)) + # print(json_dumps(req, sort_keys=True, indent=4)) # show_works(req) pass -sys.exit(0) +# sys.exit(0) """ # reqs = get_requests() @@ -126,14 +126,14 @@ def show_works(req): """ -tfs = get_transforms(request_id=219) +tfs = get_transforms(request_id=230) for tf in tfs: # print(tf) # print(tf['transform_metadata']['work'].to_dict()) - # print(json_dumps(tf, sort_keys=True, indent=4)) + print(json_dumps(tf, sort_keys=True, indent=4)) pass -# sys.exit(0) +sys.exit(0) """ msgs = retrieve_messages(workload_id=25972557) diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index ee6596d2..8c532214 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -31,7 +31,7 @@ def migrate(): cm1 = ClientManager(host=dev_host) # reqs = cm1.get_requests(request_id=290) - old_request_id = 215 + old_request_id = 225 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 # for old_request_id in [200]: # noqa E115 @@ -39,6 +39,8 @@ def migrate(): reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) cm2 = ClientManager(host=dev_host) + # print(reqs) + for req in reqs: req = convert_old_req_2_workflow_req(req) workflow = req['request_metadata']['workflow'] diff --git a/monitor/conf.js b/monitor/conf.js index 39e1faff..bdcda0bf 100644 --- a/monitor/conf.js +++ b/monitor/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus768.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus768.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus768.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus768.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus768.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus768.cern.ch:443/idds/monitor/null/null/false/false/true" } From be21ceb5f7cb6a82bb50b007da5d89319df67f78 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 6 Dec 2021 13:32:27 +0100 Subject: [PATCH 156/156] new version 0.9.1 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/version.py | 2 +- website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index e3a1b9c6..0ea0616c 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 638fe0cc..41746afd 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -11,5 +11,5 @@ dependencies: - nose # nose test tools - rucio-clients - rucio-clients-atlas - - idds-common==0.9.0 - - idds-workflow==0.9.0 \ No newline at end of file + - idds-common==0.9.1 + - idds-workflow==0.9.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index e3a1b9c6..0ea0616c 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 48d00e77..d161c850 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - tabulate - - idds-common==0.9.0 - - idds-workflow==0.9.0 \ No newline at end of file + - idds-common==0.9.1 + - idds-workflow==0.9.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index e3a1b9c6..0ea0616c 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index c802f308..881c1878 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index ba324cd8..55d1d1fd 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.9.0 - - idds-workflow==0.9.0 \ No newline at end of file + - idds-common==0.9.1 + - idds-workflow==0.9.1 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index e3a1b9c6..0ea0616c 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index adbc8bbb..2be03b29 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -22,6 +22,6 @@ dependencies: - recommonmark # use Markdown with Sphinx - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - - idds-common==0.9.0 - - idds-workflow==0.9.0 - - idds-client==0.9.0 + - idds-common==0.9.1 + - idds-workflow==0.9.1 + - idds-client==0.9.1 \ No newline at end of file diff --git a/monitor/version.py b/monitor/version.py index e3a1b9c6..0ea0616c 100644 --- a/monitor/version.py +++ b/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/website/version.py b/website/version.py index e3a1b9c6..0ea0616c 100644 --- a/website/version.py +++ b/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index e3a1b9c6..0ea0616c 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.9.0" +release_version = "0.9.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 4d9a5344..3834740c 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.9.0 \ No newline at end of file + - idds-common==0.9.1 \ No newline at end of file