Merge pull request #117 from HSF/dev

improve docker image

Dockerfile

@@ -25,9 +25,15 @@ RUN yum upgrade -y && \
     yum clean all && \
     rm -rf /var/cache/yum
 
-RUN yum install -y httpd.x86_64 conda gridsite mod_ssl.x86_64 httpd-devel.x86_64 gcc.x86_64 supervisor.noarch fetch-crl.noarch lcg-CA postgresql postgresql-contrib postgresql-static postgresql-libs postgresql-devel && \
-    yum clean all && \
-    rm -rf /var/cache/yum
+# RUN yum install -y httpd.x86_64 conda gridsite mod_ssl.x86_64 httpd-devel.x86_64 gcc.x86_64 supervisor.noarch fetch-crl.noarch lcg-CA postgresql postgresql-contrib postgresql-static postgresql-libs postgresql-devel && \
+#     yum clean all && \
+#     rm -rf /var/cache/yum
+RUN yum install -y httpd.x86_64 conda gridsite mod_ssl.x86_64 httpd-devel.x86_64 gcc.x86_64 supervisor.noarch fetch-crl.noarch lcg-CA redis && \
+yum clean all && \
+rm -rf /var/cache/yum
+
+RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
+RUN yum install -y postgresql14
 
 # RUN curl http://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo -o /etc/yum.repos.d/EGI-trustanchors.repo
 RUN curl https://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo -o /etc/yum.repos.d/EGI-trustanchors.repo

environment.yml

@@ -0,0 +1,20 @@
+name: iDDS
+dependencies:
+- python==3.6
+- pip
+- pip:
+  - requests                         # requests
+  - SQLAlchemy                       # db orm
+  - urllib3                          # url connections
+  - retrying                         # retrying behavior
+  - mysqlclient                      # mysql python client
+  #- web.py                           # web service
+  - futures                          # multiple process/threads
+  - stomp.py                         # Messaging broker client
+  - cx-Oracle                        # Oralce client
+  - unittest2        # unit test tool
+  - pep8             # checks for PEP8 code style compliance
+  - flake8           # Wrapper around PyFlakes&pep8
+  - pytest           # python testing tool
+  - nose             # nose test tools
+

main/lib/idds/agents/clerk/clerk.py

@@ -508,16 +508,16 @@ def handle_update_request_real(self, req, event):
 
         req_status = RequestStatus.Transforming
         if wf.is_terminated():
-            if wf.is_finished():
+            if wf.is_finished(synchronize=False):
                 req_status = RequestStatus.Finished
             else:
                 if to_abort and not to_abort_transform_id:
                     req_status = RequestStatus.Cancelled
-                elif wf.is_expired():
+                elif wf.is_expired(synchronize=False):
                     req_status = RequestStatus.Expired
-                elif wf.is_subfinished():
+                elif wf.is_subfinished(synchronize=False):
                     req_status = RequestStatus.SubFinished
-                elif wf.is_failed():
+                elif wf.is_failed(synchronize=False):
                     req_status = RequestStatus.Failed
                 else:
                     req_status = RequestStatus.Failed

main/lib/idds/tests/core_tests.py

@@ -1,7 +1,7 @@
 import sys
 import datetime
 
-from idds.common.utils import json_dumps                 # noqa F401
+from idds.common.utils import json_dumps, setup_logging                 # noqa F401
 from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking          # noqa F401
 from idds.core.requests import get_requests              # noqa F401
 from idds.core.messages import retrieve_messages         # noqa F401
@@ -11,6 +11,11 @@
 from idds.core import transforms as core_transforms      # noqa F401
 from idds.orm.contents import get_input_contents
 from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old     # noqa F401
+from idds.workflowv2.workflow import Workflow            # noqa F401
+from idds.workflowv2.work import Work                    # noqa F401
+
+
+setup_logging(__name__)
 
 
 def release_inputs_test():
@@ -102,6 +107,34 @@ def show_works(req):
     print(work_ids)
 
 
+def print_workflow(workflow, layers=0):
+    prefix = " " * layers * 4
+    for run in workflow.runs:
+        print(prefix + "run: " + str(run) + ", has_loop_condition: " + str(workflow.runs[run].has_loop_condition()))
+        if workflow.runs[run].has_loop_condition():
+            print(prefix + " Loop condition: %s" % json_dumps(workflow.runs[run].loop_condition, sort_keys=True, indent=4))
+        for work_id in workflow.runs[run].works:
+            print(prefix + " " + str(work_id) + " " + str(type(workflow.runs[run].works[work_id])))
+            if type(workflow.runs[run].works[work_id]) in [Workflow]:
+                print(prefix + "   parent_num_run: " + workflow.runs[run].works[work_id].parent_num_run + ", num_run: " + str(workflow.runs[run].works[work_id].num_run))
+                print_workflow(workflow.runs[run].works[work_id], layers=layers + 1)
+            # elif type(workflow.runs[run].works[work_id]) in [Work]:
+            else:
+                print(prefix + "   " + workflow.runs[run].works[work_id].task_name + ", num_run: " + str(workflow.runs[run].works[work_id].num_run))
+
+
+def print_workflow_template(workflow, layers=0):
+    prefix = " " * layers * 4
+    print(prefix + str(workflow.template.internal_id) + ", has_loop_condition: " + str(workflow.template.has_loop_condition()))
+    for work_id in workflow.template.works:
+        print(prefix + " " + str(work_id) + " " + str(type(workflow.template.works[work_id])))
+        if type(workflow.template.works[work_id]) in [Workflow]:
+            print(prefix + "   parent_num_run: " + str(workflow.template.works[work_id].parent_num_run) + ", num_run: " + str(workflow.template.works[work_id].num_run))
+            print_workflow_template(workflow.template.works[work_id], layers=layers + 1)
+        else:
+            print(prefix + "   " + workflow.template.works[work_id].task_name + ", num_run: " + str(workflow.template.works[work_id].num_run))
+
+
 # 283511, 283517
 # reqs = get_requests(request_id=599, with_detail=True, with_metadata=True)
 # reqs = get_requests(request_id=283511, with_request=True, with_detail=False, with_metadata=True)
@@ -115,12 +148,15 @@ def show_works(req):
 # reqs = get_requests(request_id=370028, with_request=True, with_detail=False, with_metadata=True)
 # reqs = get_requests(request_id=370400, with_request=True, with_detail=False, with_metadata=True)
 # reqs = get_requests(request_id=371204, with_request=True, with_detail=False, with_metadata=True)
-reqs = get_requests(request_id=372678, with_request=True, with_detail=False, with_metadata=True)
+# reqs = get_requests(request_id=372678, with_request=True, with_detail=False, with_metadata=True)
+# reqs = get_requests(request_id=373602, with_request=True, with_detail=False, with_metadata=True)
+# reqs = get_requests(request_id=376086, with_request=True, with_detail=False, with_metadata=True)
+reqs = get_requests(request_id=380474, with_request=True, with_detail=False, with_metadata=True)
 for req in reqs:
     # print(req['request_id'])
-    print(req)
+    # print(req)
     # print(rets)
-    print(json_dumps(req, sort_keys=True, indent=4))
+    # print(json_dumps(req, sort_keys=True, indent=4))
     # show_works(req)
     pass
     workflow = req['request_metadata']['workflow']
@@ -129,16 +165,28 @@ def show_works(req):
     # print(workflow.runs["1"])
     # print(json_dumps(workflow.runs["1"], sort_keys=True, indent=4))
 
-    print(workflow.runs["1"].works.keys())
-    print(workflow.runs["1"].works["048a1811"])
+    # print(workflow.runs["1"].works.keys())
+    # print(workflow.runs["1"].has_loop_condition())
+    # print(workflow.runs["1"].works["7aa1ec08"])
     # print(json_dumps(workflow.runs["1"].works["048a1811"], indent=4))
-    print(workflow.runs["1"].works["048a1811"].runs.keys())
-    print(json_dumps(workflow.runs["1"].works["048a1811"].runs["2"], indent=4))
+    # print(workflow.runs["1"].works["7aa1ec08"].runs.keys())
+    # print(workflow.runs["1"].works["7aa1ec08"].runs["1"].has_loop_condition())
+    # print(workflow.runs["1"].works["7aa1ec08"].runs["1"].works.keys())
+
+    # print(json_dumps(workflow.runs["1"].works["7aa1ec08"].runs["1"], indent=4))
     if hasattr(workflow, 'get_relation_map'):
         # print(json_dumps(workflow.get_relation_map(), sort_keys=True, indent=4))
         pass
 
-sys.exit(0)
+    print("workflow")
+    print_workflow(workflow)
+    print("workflow template")
+    print_workflow_template(workflow)
+
+    # workflow.sync_works()
+
+# sys.exit(0)
+
 
 """
 # reqs = get_requests()
@@ -155,7 +203,7 @@ def show_works(req):
 """
 
 
-tfs = get_transforms(request_id=370028)
+tfs = get_transforms(request_id=380474)
 # tfs = get_transforms(transform_id=350723)
 for tf in tfs:
     # print(tf)
@@ -166,6 +214,7 @@ def show_works(req):
     print(tf['transform_metadata']['work_name'])
     print(tf['transform_metadata']['work'].num_run)
     print(tf['transform_metadata']['work'].task_name)
+    print(tf['transform_metadata']['work'].output_data)
     pass
 
 sys.exit(0)
@@ -190,7 +239,7 @@ def show_works(req):
 sys.exit(0)
 """
 
-prs = get_processings(request_id=350723)
+prs = get_processings(request_id=373602)
 # prs = get_processings(transform_id=350723)
 i = 0
 for pr in prs:

main/lib/idds/tests/retry_processing.py

@@ -0,0 +1,9 @@
+import sys
+
+from pandaclient import Client
+
+task_id = sys.argv[1]
+
+status, output = Client.retryTask(task_id)
+print(status)
+print(output)

main/lib/idds/tests/test_migrate_requests.py

@@ -44,6 +44,7 @@ def migrate():
     # old_request_id = 349
     old_request_id = 2400
     old_request_id = 371204
+    old_request_id = 372930
 
     # for old_request_id in [152]:
     # for old_request_id in [60]:    # noqa E115

main/tools/env/setup_idds.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export IDDS_HOST=https://aipanda160.cern.ch:443/idds
+export IDDS_LOCAL_CONFIG_ROOT=~/.idds
+# export IDDS_CONFIG=
+export IDDS_AUTH_TYPE=oidc
+# export IDDS_OIDC_TOKEN=~/.idds/.token
+export IDDS_VO=Rubin
+export IDDS_AUTH_NO_VERIFY=1
+