Merge pull request #183 from HSF/flin

worker adjuster: fix pull n_activated when no job in job_stats; v0.2.29
HSF · Apr 19, 2023 · ed08b29 · ed08b29
2 parents 55e100d + eaaac2a
commit ed08b29
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 13 deletions.
diff --git a/pandaharvester/commit_timestamp.py b/pandaharvester/commit_timestamp.py
@@ -1 +1 @@
-timestamp = "06-04-2023 13:05:57 on lancium (by mightqxc)"
+timestamp = "19-04-2023 13:49:56 on flin (by mightqxc)"
diff --git a/pandaharvester/harvesterbody/worker_adjuster.py b/pandaharvester/harvesterbody/worker_adjuster.py
@@ -41,9 +41,7 @@ def define_num_workers(self, static_num_workers, site_name):
 
             # get job statistics
             job_stats = self.dbProxy.get_cache("job_statistics.json", None)
-            if job_stats is None:
-                job_stats = dict()
-            else:
+            if job_stats is not None:
                 job_stats = job_stats.data
 
             # define num of new workers
@@ -147,16 +145,22 @@ def define_num_workers(self, static_num_workers, site_name):
                                     max_queued_workers = maxQueuedWorkers_slave
 
                             elif queue_config.mapType == 'NoJob': # for pull mode, limit to activated jobs
-                                # limit the queue to the number of activated jobs to avoid empty pilots
-                                try:
-                                    n_activated = max(job_stats[queue_name]['activated'], 1) # avoid no activity queues
-                                    queue_limit = max_queued_workers
-                                    max_queued_workers = min(n_activated, max_queued_workers)
-                                    tmp_log.debug('limiting max_queued_workers to min(n_activated={0}, queue_limit={1})'.
-                                                 format(n_activated, queue_limit))
-                                except KeyError:
+                                if job_stats is None:
                                     tmp_log.warning('n_activated not defined, defaulting to configured queue limits')
                                     pass
+                                else:
+                                    # limit the queue to the number of activated jobs to avoid empty pilots
+                                    try:
+                                        n_activated = max(job_stats[queue_name]['activated'], 1) # avoid no activity queues
+                                    except KeyError:
+                                        # zero job in the queue
+                                        tmp_log.debug('no job in queue')
+                                        n_activated = 1
+                                    finally:
+                                        queue_limit = max_queued_workers
+                                        max_queued_workers = min(n_activated, max_queued_workers)
+                                        tmp_log.debug('limiting max_queued_workers to min(n_activated={0}, queue_limit={1})'.
+                                                    format(n_activated, queue_limit))
 
                             if max_queued_workers is None:  # no value found, use default value
                                 max_queued_workers = 1

diff --git a/pandaharvester/panda_pkg_info.py b/pandaharvester/panda_pkg_info.py
@@ -1 +1 @@
-release_version = "0.2.28"
+release_version = "0.2.29"
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		timestamp = "06-04-2023 13:05:57 on lancium (by mightqxc)"
		timestamp = "19-04-2023 13:49:56 on flin (by mightqxc)"
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		release_version = "0.2.28"
		release_version = "0.2.29"