Add silent_on_errors argument to Task.delete_artifacts (default False)

clearml/backend_interface/task/task.py

@@ -923,11 +923,11 @@ def _delete(
 
         return task_deleted
 
-    def _delete_uri(self, uri):
-        # type: (str) -> bool
+    def _delete_uri(self, uri, silent=False):
+        # type: (str, bool) -> bool
         # noinspection PyBroadException
         try:
-            deleted = StorageHelper.get(uri).delete(uri)
+            deleted = StorageHelper.get(uri).delete(uri, silent=silent)
             if deleted:
                 self.log.debug("Deleted file: {}".format(uri))
                 return True
@@ -1539,29 +1539,38 @@ def _add_artifacts(self, artifacts_list):
                 self._edit(execution=execution)
         return self.data.execution.artifacts or []
 
-    def delete_artifacts(self, artifact_names, raise_on_errors=True, delete_from_storage=True):
-        # type: (Sequence[str], bool, bool) -> bool
+    def delete_artifacts(self, artifact_names, raise_on_errors=True, delete_from_storage=True, silent_on_errors=False):
+        # type: (Sequence[str], bool, bool, bool) -> bool
         """
         Delete a list of artifacts, by artifact name, from the Task.
 
         :param list artifact_names: list of artifact names
         :param bool raise_on_errors: if True, do not suppress connectivity related exceptions
         :param bool delete_from_storage: If True, try to delete the actual
             file from the external storage (e.g. S3, GS, Azure, File Server etc.)
+        :param silent_on_errors: If True, do not log connectivity related errors
 
         :return: True if successful
         """
-        return self._delete_artifacts(artifact_names, raise_on_errors, delete_from_storage)
+        return self._delete_artifacts(
+            artifact_names=artifact_names,
+            raise_on_errors=raise_on_errors,
+            delete_from_storage=delete_from_storage,
+            silent_on_errors=silent_on_errors
+        )
 
-    def _delete_artifacts(self, artifact_names, raise_on_errors=False, delete_from_storage=True):
-        # type: (Sequence[str], bool, bool) -> bool
+    def _delete_artifacts(
+        self, artifact_names, raise_on_errors=False, delete_from_storage=True, silent_on_errors=False
+    ):
+        # type: (Sequence[str], bool, bool, bool) -> bool
         """
         Delete a list of artifacts, by artifact name, from the Task.
 
         :param list artifact_names: list of artifact names
         :param bool raise_on_errors: if True, do not suppress connectivity related exceptions
         :param bool delete_from_storage: If True, try to delete the actual
-        file from the external storage (e.g. S3, GS, Azure, File Server etc.)
+            file from the external storage (e.g. S3, GS, Azure, File Server etc.)
+        :param silent_on_errors: If True, do not log connectivity related errors
 
         :return: True if successful
         """
@@ -1605,7 +1614,7 @@ def _delete_artifacts(self, artifact_names, raise_on_errors=False, delete_from_s
         if uris:
             for i, (artifact, uri) in enumerate(zip(artifact_names, uris)):
                 # delete the actual file from storage, and raise if error and needed
-                if uri and not self._delete_uri(uri) and raise_on_errors:
+                if uri and not self._delete_uri(uri, silent=silent_on_errors) and raise_on_errors:
                     remaining_uris = {name: uri for name, uri in zip(artifact_names[i + 1:], uris[i + 1:])}
                     raise ArtifactUriDeleteError(artifact=artifact, uri=uri, remaining_uris=remaining_uris)
 

clearml/storage/helper.py

@@ -254,8 +254,10 @@ def delete_object(self, obj, *args, **kwargs):
         container = self._containers[obj.container_name]
         res = container.session.delete(obj.url, headers=container.get_headers(obj.url))
         if res.status_code != requests.codes.ok:
-            self.get_logger().warning('Failed deleting object %s (%d): %s' % (
-                obj.object_name, res.status_code, res.text))
+            if not kwargs.get("silent", False):
+                self.get_logger().warning(
+                    'Failed deleting object %s (%d): %s' % (obj.object_name, res.status_code, res.text)
+                )
             return False
         return True
 
@@ -908,7 +910,8 @@ def delete_object(self, object, **kwargs):
             except ImportError:
                 pass
             name = getattr(object, "name", "")
-            self.get_logger().warning("Failed deleting object {}: {}".format(name, ex))
+            if not kwargs.get("silent", False):
+                self.get_logger().warning("Failed deleting object {}: {}".format(name, ex))
             return False
 
         return not object.exists()
@@ -2797,9 +2800,9 @@ def download_as_nparray(self, remote_path, chunk_size=None):
         except Exception as e:
             self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e)))
 
-    def delete(self, path):
+    def delete(self, path, silent=False):
         path = self._canonize_url(path)
-        return self._driver.delete_object(self.get_object(path))
+        return self._driver.delete_object(self.get_object(path), silent=silent)
 
     def check_write_permissions(self, dest_path=None):
         # create a temporary file, then delete it