Merge pull request #69 from wguanicedew/dev

Dev

atlas/lib/idds/atlas/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.10.3"
+release_version = "0.10.4"

atlas/lib/idds/atlas/workflow/atlaspandawork.py

@@ -392,7 +392,7 @@ def create_processing(self, input_output_maps=[]):
 
     def submit_panda_task(self, processing):
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             proc = processing['processing_metadata']['processing']
             task_param = proc.processing_metadata['task_param']
@@ -436,7 +436,7 @@ def submit_processing(self, processing):
 
     def poll_panda_task_status(self, processing):
         if 'processing' in processing['processing_metadata']:
-            from pandatools import Client
+            from pandaclient import Client
 
             proc = processing['processing_metadata']['processing']
             status, task_status = Client.getTaskStatus(proc.workload_id)
@@ -468,7 +468,7 @@ def get_processing_status_from_panda_status(self, task_status):
         return processing_status
 
     def get_panda_task_id(self, processing):
-        from pandatools import Client
+        from pandaclient import Client
 
         start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=10)
         start_time = start_time.strftime('%Y-%m-%d %H:%M:%S')
@@ -494,7 +494,7 @@ def get_panda_task_id(self, processing):
     def poll_panda_task(self, processing=None, input_output_maps=None):
         task_id = None
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             if processing:
                 proc = processing['processing_metadata']['processing']
@@ -534,7 +534,7 @@ def poll_panda_task(self, processing=None, input_output_maps=None):
     def kill_processing(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id
                 # task_id = processing['processing_metadata']['task_id']
@@ -547,7 +547,7 @@ def kill_processing(self, processing):
     def kill_processing_force(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id
                 # task_id = processing['processing_metadata']['task_id']
@@ -560,7 +560,7 @@ def kill_processing_force(self, processing):
     def reactivate_processing(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 # task_id = processing['processing_metadata']['task_id']
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id

atlas/lib/idds/atlas/workflowv2/atlasdagwork.py

@@ -347,7 +347,7 @@ def create_processing(self, input_output_maps):
 
     def submit_panda_task(self, processing):
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             task_param = processing['processing_metadata']['task_param']
             return_code = Client.insertTaskParams(task_param, verbose=True)

atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py

@@ -296,7 +296,7 @@ def get_rucio_download_client(self):
     def poll_panda_task_output(self, processing=None, input_output_maps=None):
         task_id = None
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             if processing:
                 output_metadata = {}

atlas/lib/idds/atlas/workflowv2/atlaspandawork.py

@@ -454,7 +454,7 @@ def create_processing(self, input_output_maps=[]):
 
     def submit_panda_task(self, processing):
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             proc = processing['processing_metadata']['processing']
             task_param = proc.processing_metadata['task_param']
@@ -498,7 +498,7 @@ def submit_processing(self, processing):
 
     def poll_panda_task_status(self, processing):
         if 'processing' in processing['processing_metadata']:
-            from pandatools import Client
+            from pandaclient import Client
 
             proc = processing['processing_metadata']['processing']
             status, task_status = Client.getTaskStatus(proc.workload_id)
@@ -530,7 +530,7 @@ def get_processing_status_from_panda_status(self, task_status):
         return processing_status
 
     def get_panda_task_id(self, processing):
-        from pandatools import Client
+        from pandaclient import Client
 
         start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=10)
         start_time = start_time.strftime('%Y-%m-%d %H:%M:%S')
@@ -556,7 +556,7 @@ def get_panda_task_id(self, processing):
     def poll_panda_task(self, processing=None, input_output_maps=None):
         task_id = None
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             if processing:
                 proc = processing['processing_metadata']['processing']
@@ -596,7 +596,7 @@ def poll_panda_task(self, processing=None, input_output_maps=None):
     def kill_processing(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id
                 # task_id = processing['processing_metadata']['task_id']
@@ -609,7 +609,7 @@ def kill_processing(self, processing):
     def kill_processing_force(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id
                 # task_id = processing['processing_metadata']['task_id']
@@ -622,7 +622,7 @@ def kill_processing_force(self, processing):
     def reactivate_processing(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 # task_id = processing['processing_metadata']['task_id']
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id

atlas/tools/env/environment.yml

@@ -11,5 +11,5 @@ dependencies:
   - nose             # nose test tools
   - rucio-clients
   - rucio-clients-atlas
-  - idds-common==0.10.3
-  - idds-workflow==0.10.3
+  - idds-common==0.10.4
+  - idds-workflow==0.10.4

client/lib/idds/client/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.10.3"
+release_version = "0.10.4"

client/tools/env/environment.yml

@@ -14,5 +14,5 @@ dependencies:
   - pytest           # python testing tool
   - nose             # nose test tools
   - tabulate
-  - idds-common==0.10.3
-  - idds-workflow==0.10.3
+  - idds-common==0.10.4
+  - idds-workflow==0.10.4

common/lib/idds/common/constants.py

@@ -6,7 +6,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0OA
 #
 # Authors:
-# - Wen Guan, <[email protected]>, 2019 - 2020
+# - Wen Guan, <[email protected]>, 2019 - 2022
 
 """
 Constants.
@@ -314,6 +314,7 @@ class ProcessingStatus(IDDSEnum):
     TimeOut = 23
     ToFinish = 24
     ToForceFinish = 25
+    Broken = 26
 
 
 class ProcessingLocking(IDDSEnum):

common/lib/idds/common/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.10.3"
+release_version = "0.10.4"

doma/lib/idds/doma/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2020 - 2021
 
 
-release_version = "0.10.3"
+release_version = "0.10.4"

doma/lib/idds/doma/workflow/domapandawork.py

@@ -430,7 +430,7 @@ def create_processing(self, input_output_maps=[]):
 
     def submit_panda_task(self, processing):
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             proc = processing['processing_metadata']['processing']
             task_param = proc.processing_metadata['task_param']
@@ -462,7 +462,7 @@ def submit_processing(self, processing):
                 proc.submitted_at = datetime.datetime.utcnow()
 
     def get_panda_task_id(self, processing):
-        from pandatools import Client
+        from pandaclient import Client
 
         start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=10)
         start_time = start_time.strftime('%Y-%m-%d %H:%M:%S')
@@ -487,7 +487,7 @@ def get_panda_task_id(self, processing):
 
     def poll_panda_task_status(self, processing):
         if 'processing' in processing['processing_metadata']:
-            from pandatools import Client
+            from pandaclient import Client
 
             proc = processing['processing_metadata']['processing']
             status, task_status = Client.getTaskStatus(proc.workload_id)
@@ -641,7 +641,7 @@ def get_update_contents_from_map_id(self, map_id, input_output_maps, job_info):
 
     def map_panda_ids(self, unregistered_job_ids, input_output_maps):
         self.logger.debug("map_panda_ids, unregistered_job_ids[:10]: %s" % str(unregistered_job_ids[:10]))
-        from pandatools import Client
+        from pandaclient import Client
 
         # updated_map_ids = []
         full_update_contents = []
@@ -669,7 +669,7 @@ def map_panda_ids(self, unregistered_job_ids, input_output_maps):
 
     def get_status_changed_contents(self, unterminated_job_ids, input_output_maps, panda_id_to_map_ids):
         self.logger.debug("get_status_changed_contents, unterminated_job_ids[:10]: %s" % str(unterminated_job_ids[:10]))
-        from pandatools import Client
+        from pandaclient import Client
 
         full_update_contents = []
         chunksize = 2000
@@ -698,7 +698,7 @@ def get_final_update_contents(self, input_output_maps):
     def poll_panda_task_old(self, processing=None, input_output_maps=None):
         task_id = None
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             jobs_ids = None
             if processing:
@@ -769,7 +769,7 @@ def poll_panda_task_old(self, processing=None, input_output_maps=None):
     def poll_panda_jobs(self, job_ids):
         job_ids = list(job_ids)
         self.logger.debug("poll_panda_jobs, poll_panda_jobs_chunk_size: %s, job_ids[:10]: %s" % (self.poll_panda_jobs_chunk_size, str(job_ids[:10])))
-        from pandatools import Client
+        from pandaclient import Client
 
         # updated_map_ids = []
         inputname_jobid_map = {}
@@ -874,7 +874,7 @@ def get_update_contents(self, inputnames, inputname_mapid_map, inputname_jobid_m
     def poll_panda_task(self, processing=None, input_output_maps=None):
         task_id = None
         try:
-            from pandatools import Client
+            from pandaclient import Client
 
             if processing:
                 proc = processing['processing_metadata']['processing']
@@ -971,7 +971,7 @@ def poll_panda_task(self, processing=None, input_output_maps=None):
     def kill_processing(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id
                 # task_id = processing['processing_metadata']['task_id']
@@ -984,7 +984,7 @@ def kill_processing(self, processing):
     def kill_processing_force(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id
                 # task_id = processing['processing_metadata']['task_id']
@@ -997,7 +997,7 @@ def kill_processing_force(self, processing):
     def reactivate_processing(self, processing):
         try:
             if processing:
-                from pandatools import Client
+                from pandaclient import Client
                 # task_id = processing['processing_metadata']['task_id']
                 proc = processing['processing_metadata']['processing']
                 task_id = proc.workload_id