Merge pull request #30 from eth-cscs/poll-empty

Poll in a loop to avoid empty lists as result

firecrestspawner/spawner.py

@@ -209,6 +209,22 @@ async def get_firecrest_client(self):
         client.timeout = 30
         return client
 
+    async def firecrest_poll(self):
+        """Helper function to poll jobs.
+
+        This is needed in cases that the scheduler is slow updating
+        its database which could make the result of ``client.poll``
+        to be an empty list
+        """
+        client = await self.get_firecrest_client()
+        poll_result = []
+
+        while poll_result == []:
+            poll_result = await client.poll(self.host, [self.job_id])
+            print(poll_result)
+
+        return poll_result
+
     async def _get_batch_script(self, **subvars):
         """Format batch script from vars"""
         # Could be overridden by subclasses, but mainly useful for testing
@@ -258,9 +274,7 @@ async def query_job_status(self):
         self.log.debug(f'Spawner querying job {self.job_id}')
 
         try:
-            client = await self.get_firecrest_client()
-            self.log.debug('firecREST: Polling job')
-            poll_result = await client.poll(self.host, [self.job_id])
+            poll_result = await self.firecrest_poll()
             self.log.debug(f"[client.poll] [query_job_status] {poll_result}")
             state = poll_result[0]['state']
             nodelist = hostlist.expand_hostlist(poll_result[0]['nodelist'])
@@ -502,9 +516,11 @@ async def state_gethost(self):
         if self.custom_state_gethost:
             return await self.custom_state_gethost(self)
 
-        client = await self.get_firecrest_client()
-        poll_result = await client.poll(self.host, [self.job_id])
+        poll_result = await self.firecrest_poll()
         self.log.debug(f"[client.poll] [state_gethost] {poll_result}")
+
+        # this function is called only when the job has been allocated,
+        # then ``nodelist`` won't be ``[]``
         host = hostlist.expand_hostlist(poll_result[0]['nodelist'])[0]
         return self.node_name_template.format(host)