Prune run containers at one other place (#1102)

* Update docker dependency to 3.7.0

* Get rid of non-existent tracked docker images and handle empty str in
local run manager container pruning

worker/codalabworker/local_run/docker_image_manager.py

@@ -108,6 +108,9 @@ def _cleanup(self):
                         self._docker.images.remove(tag)
                     # if we successfully removed the image also remove its cache entry
                     del self._image_cache[entry_to_remove.digest]
+                except docker.errors.NotFound:
+                    # image doesn't exist anymore for some reason, stop tracking it
+                    del self._image_cache[entry_to_remove.digest]
                 except docker.errors.APIError as err:
                     # Maybe we can't delete this image because its container is still running
                     # (think a run that takes 4 days so this is the oldest image but still in use)

worker/codalabworker/local_run/local_run_manager.py

@@ -174,6 +174,18 @@ def process_runs(self):
                 self._runs[bundle_uuid] = self._run_state_manager.transition(run_state)
 
             # filter out finished runs
+            finished_container_ids = [
+                run.container
+                for run in self._runs.values()
+                if (run.stage == LocalRunStage.FINISHED or run.stage == LocalRunStage.FINALIZING)
+                and run.container_id is not None
+            ]
+            for container_id in finished_container_ids:
+                try:
+                    container = self._docker.containers.get(container_id)
+                    container.remove(force=True)
+                except (docker.errors.NotFound, docker.errors.NullResource):
+                    pass
             self._runs = {k: v for k, v in self._runs.items() if v.stage != LocalRunStage.FINISHED}
 
     def create_run(self, bundle, resources):

worker/requirements.txt

@@ -1,5 +1,5 @@
 bottle==0.12.9
-docker==3.5.0
+docker==3.7.0
 marshmallow-jsonapi==0.15.1
 marshmallow==2.6.1
 setuptools>=40.0.0