From afed708d10d9bb0cf7ac46dda67737dc7aea8168 Mon Sep 17 00:00:00 2001 From: Giacomo Alzetta Date: Wed, 15 Jan 2025 17:52:22 +0100 Subject: [PATCH] Add option to preserve trailing slash on deletes Most object storages support having a slash in the object name and rohmu should be able to delete such objects. Let the default behaviour the same but introduce an option `preserve_trailing_slash` which allows to control whether the trailing slash in the key are stripped or not. Added test coverage for `delete_key` and `delete_keys` methods for all transfer types (several had no coverage of it). --- rohmu/object_storage/azure.py | 6 ++- rohmu/object_storage/base.py | 10 ++-- rohmu/object_storage/google.py | 4 +- rohmu/object_storage/local.py | 10 ++-- rohmu/object_storage/s3.py | 18 +++++-- rohmu/object_storage/sftp.py | 6 ++- rohmu/object_storage/swift.py | 4 +- test/object_storage/test_azure.py | 71 +++++++++++++++++++++++++- test/object_storage/test_google.py | 77 ++++++++++++++++++++++++++++ test/object_storage/test_local.py | 72 +++++++++++++++++++++++++- test/object_storage/test_s3.py | 38 ++++++++++++-- test/object_storage/test_sftp.py | 81 +++++++++++++++++++++++++++++- test/object_storage/test_swift.py | 73 ++++++++++++++++++++++++++- 13 files changed, 439 insertions(+), 31 deletions(-) diff --git a/rohmu/object_storage/azure.py b/rohmu/object_storage/azure.py index fcc89a15..0281e456 100644 --- a/rohmu/object_storage/azure.py +++ b/rohmu/object_storage/azure.py @@ -301,8 +301,10 @@ def _iter_key(self, *, path: str, with_metadata: bool, deep: bool) -> Iterator[I }, ) - def delete_key(self, key: str) -> None: - path = self.format_key_for_backend(key, remove_slash_prefix=True) + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: + path = self.format_key_for_backend( + key, remove_slash_prefix=True, trailing_slash=preserve_trailing_slash and key.endswith("/") + ) self.log.debug("Deleting key: %r", path) try: blob_client = self.get_blob_service_client().get_blob_client(container=self.container_name, blob=path) diff --git a/rohmu/object_storage/base.py b/rohmu/object_storage/base.py index 95766929..ec301115 100644 --- a/rohmu/object_storage/base.py +++ b/rohmu/object_storage/base.py @@ -255,20 +255,20 @@ def format_key_from_backend(self, key: str) -> str: raise StorageError(f"Key {repr(key)} does not start with expected prefix {repr(self.prefix)}") return key[len(self.prefix) :] - def delete_key(self, key: str) -> None: + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: raise NotImplementedError - def delete_keys(self, keys: Collection[str]) -> None: + def delete_keys(self, keys: Collection[str], preserve_trailing_slash: bool = False) -> None: """Delete specified keys""" for key in keys: - self.delete_key(key) + self.delete_key(key, preserve_trailing_slash=preserve_trailing_slash) - def delete_tree(self, key: str) -> None: + def delete_tree(self, key: str, preserve_trailing_slash: bool = False) -> None: """Delete all keys under given root key. Basic implementation works by just listing all available keys and deleting them individually but storage providers can implement more efficient logic.""" self.log.debug("Deleting tree: %r", key) names = [item["name"] for item in self.list_path(key, with_metadata=False, deep=True)] - self.delete_keys(names) + self.delete_keys(names, preserve_trailing_slash=preserve_trailing_slash) def get_contents_to_file( self, key: str, filepath_to_store_to: AnyPath, *, progress_callback: ProgressProportionCallbackType = None diff --git a/rohmu/object_storage/google.py b/rohmu/object_storage/google.py index c6e05043..e93f6fa3 100644 --- a/rohmu/object_storage/google.py +++ b/rohmu/object_storage/google.py @@ -452,8 +452,8 @@ def initial_op(domain: Any) -> HttpRequest: else: raise NotImplementedError(property_name) - def delete_key(self, key: str) -> None: - path = self.format_key_for_backend(key) + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: + path = self.format_key_for_backend(key, trailing_slash=preserve_trailing_slash and key.endswith("/")) self.log.debug("Deleting key: %r", path) with self._object_client(not_found=path) as clob: # https://googleapis.github.io/google-api-python-client/docs/dyn/storage_v1.objects.html#delete diff --git a/rohmu/object_storage/local.py b/rohmu/object_storage/local.py index 84775aea..7e60636e 100644 --- a/rohmu/object_storage/local.py +++ b/rohmu/object_storage/local.py @@ -8,7 +8,7 @@ from pathlib import Path from rohmu.common.models import StorageOperation from rohmu.common.statsd import StatsdConfig -from rohmu.errors import ConcurrentUploadError, FileNotFoundFromStorageError +from rohmu.errors import ConcurrentUploadError, Error, FileNotFoundFromStorageError from rohmu.notifier.interface import Notifier from rohmu.object_storage.base import ( BaseTransfer, @@ -123,8 +123,10 @@ def _filter_metadata(self, metadata: Metadata) -> Metadata: def get_metadata_for_key(self, key: str) -> Metadata: return self._filter_metadata(self._get_metadata_for_key(key)) - def delete_key(self, key: str) -> None: + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: self.log.debug("Deleting key: %r", key) + if preserve_trailing_slash: + raise Error("LocalTransfer does not support preserving trailing slashes") target_path = self.format_key_for_backend(key.strip("/")) if not os.path.exists(target_path): raise FileNotFoundFromStorageError(key) @@ -137,8 +139,10 @@ def delete_key(self, key: str) -> None: os.unlink(metadata_path) self.notifier.object_deleted(key=key) - def delete_tree(self, key: str) -> None: + def delete_tree(self, key: str, preserve_trailing_slash: bool = False) -> None: self.log.debug("Deleting tree: %r", key) + if preserve_trailing_slash: + raise Error("LocalTransfer does not support preserving trailing slashes") target_path = self.format_key_for_backend(key.strip("/")) if not os.path.isdir(target_path): raise FileNotFoundFromStorageError(key) diff --git a/rohmu/object_storage/s3.py b/rohmu/object_storage/s3.py index e19fd693..a2eab84e 100644 --- a/rohmu/object_storage/s3.py +++ b/rohmu/object_storage/s3.py @@ -331,20 +331,30 @@ def _metadata_for_key(self, key: str) -> Metadata: return response["Metadata"] - def delete_key(self, key: str) -> None: - path = self.format_key_for_backend(key, remove_slash_prefix=True) + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: + path = self.format_key_for_backend( + key, remove_slash_prefix=True, trailing_slash=preserve_trailing_slash and key.endswith("/") + ) self.log.debug("Deleting key: %r", path) self._metadata_for_key(path) # check that key exists self.stats.operation(StorageOperation.delete_key) self.get_client().delete_object(Bucket=self.bucket_name, Key=path) self.notifier.object_deleted(key=key) - def delete_keys(self, keys: Collection[str]) -> None: + def delete_keys(self, keys: Collection[str], preserve_trailing_slash: bool = False) -> None: self.stats.operation(StorageOperation.delete_key, count=len(keys)) for batch in batched(keys, 1000): # Cannot delete more than 1000 objects at a time + formatted_keys = [ + self.format_key_for_backend( + k, + remove_slash_prefix=True, + trailing_slash=preserve_trailing_slash and k.endswith("/"), + ) + for k in batch + ] self.get_client().delete_objects( Bucket=self.bucket_name, - Delete={"Objects": [{"Key": self.format_key_for_backend(key, remove_slash_prefix=True)} for key in batch]}, + Delete={"Objects": [{"Key": key} for key in formatted_keys]}, ) # Note: `tree_deleted` is not used here because the operation on S3 is not atomic, i.e. # it is possible for a new object to be created after `list_objects` above diff --git a/rohmu/object_storage/sftp.py b/rohmu/object_storage/sftp.py index 8695ad50..147c2982 100644 --- a/rohmu/object_storage/sftp.py +++ b/rohmu/object_storage/sftp.py @@ -5,7 +5,7 @@ from io import BytesIO from rohmu.common.statsd import StatsdConfig -from rohmu.errors import FileNotFoundFromStorageError, InvalidConfigurationError +from rohmu.errors import Error, FileNotFoundFromStorageError, InvalidConfigurationError from rohmu.notifier.interface import Notifier from rohmu.object_storage.base import ( BaseTransfer, @@ -210,7 +210,9 @@ def copy_file( ) -> None: raise NotImplementedError - def delete_key(self, key: str) -> None: + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: + if preserve_trailing_slash: + raise Error("SftpTransfer does not support preserving trailing slashes") target_path = self.format_key_for_backend(key.strip("/")) self.log.info("Removing path: %r", target_path) diff --git a/rohmu/object_storage/swift.py b/rohmu/object_storage/swift.py index 81efa12c..e108afcc 100644 --- a/rohmu/object_storage/swift.py +++ b/rohmu/object_storage/swift.py @@ -225,8 +225,8 @@ def _delete_object_segments(self, key: str, manifest: str) -> None: with suppress(FileNotFoundFromStorageError): self._delete_object_plain(item["name"]) - def delete_key(self, key: str) -> None: - path = self.format_key_for_backend(key) + def delete_key(self, key: str, preserve_trailing_slash: bool = False) -> None: + path = self.format_key_for_backend(key, trailing_slash=preserve_trailing_slash and key.endswith("/")) self.log.debug("Deleting key: %r", path) try: headers = self.conn.head_object(self.container_name, path) diff --git a/test/object_storage/test_azure.py b/test/object_storage/test_azure.py index 6285e45d..fdb5a9b7 100644 --- a/test/object_storage/test_azure.py +++ b/test/object_storage/test_azure.py @@ -7,8 +7,8 @@ from rohmu.object_storage.azure import AzureTransfer from rohmu.object_storage.config import AzureObjectStorageConfig from tempfile import NamedTemporaryFile -from typing import Any, Optional -from unittest.mock import MagicMock, patch +from typing import Any, Optional, Union +from unittest.mock import call, MagicMock, patch import azure.storage.blob import pytest @@ -241,3 +241,70 @@ def test_create_container_str(mocker: MockerFixture) -> None: ) container_name = container_client_mock.call_args.kwargs["container_name"] assert container_name == "bucket_name" + + +@pytest.mark.parametrize( + ("key", "preserve_trailing_slash", "expected_key"), + [ + ("1", True, "test-prefix/1"), + ("2/", True, "test-prefix/2/"), + ("1", False, "test-prefix/1"), + ("2/", False, "test-prefix/2"), + ("1", None, "test-prefix/1"), + ("2/", None, "test-prefix/2"), + ], +) +def test_delete_key( + mock_get_blob_client: MagicMock, + key: str, + preserve_trailing_slash: Union[bool, None], + expected_key: str, +) -> None: + notifier = MagicMock() + transfer = AzureTransfer( + bucket_name="test_bucket", + account_name="test_account", + account_key="test_key2", + prefix="test-prefix/", + notifier=notifier, + ) + + if preserve_trailing_slash is None: + transfer.delete_key(key) + else: + transfer.delete_key(key, preserve_trailing_slash=preserve_trailing_slash) + + mock_get_blob_client.assert_has_calls( + [ + call(container="test_bucket", blob=expected_key), + call().delete_blob(), + ] + ) + + +@pytest.mark.parametrize("preserve_trailing_slash", [True, False, None]) +def test_delete_keys(mock_get_blob_client: MagicMock, preserve_trailing_slash: Union[bool, None]) -> None: + notifier = MagicMock() + transfer = AzureTransfer( + bucket_name="test_bucket", + account_name="test_account", + account_key="test_key2", + prefix="test-prefix/", + notifier=notifier, + ) + if preserve_trailing_slash is None: + transfer.delete_keys(["2", "3", "4/"]) + else: + transfer.delete_keys(["2", "3", "4/"], preserve_trailing_slash=preserve_trailing_slash) + + expected_keys = ["2", "3", "4/" if preserve_trailing_slash else "4"] + expected_calls = [] + for expected_key in expected_keys: + expected_calls.extend( + [ + call(container="test_bucket", blob=f"test-prefix/{expected_key}"), + call().delete_blob(), + ] + ) + + mock_get_blob_client.assert_has_calls(expected_calls) diff --git a/test/object_storage/test_google.py b/test/object_storage/test_google.py index 11d0644b..02825730 100644 --- a/test/object_storage/test_google.py +++ b/test/object_storage/test_google.py @@ -12,6 +12,7 @@ from rohmu.object_storage.base import IterKeyItem from rohmu.object_storage.google import GoogleTransfer, MediaIoBaseDownloadWithByteRange, Reporter from tempfile import NamedTemporaryFile +from typing import Union from unittest.mock import ANY, call, MagicMock, Mock, patch import base64 @@ -452,3 +453,79 @@ def test_error_handling() -> None: # ... and the legacy behaviour of bubbling up should not regress with pytest.raises(HttpError, match="403"): transfer._create_object_store_if_needed_unwrapped() + + +@pytest.mark.parametrize( + ("key", "preserve_trailing_slash", "expected_key"), + [ + ("1", True, "test-prefix/1"), + ("2/", True, "test-prefix/2/"), + ("1", False, "test-prefix/1"), + ("2/", False, "test-prefix/2"), + ("1", None, "test-prefix/1"), + ("2/", None, "test-prefix/2"), + ], +) +def test_delete_key(key: str, preserve_trailing_slash: Union[bool, None], expected_key: str) -> None: + notifier = MagicMock() + with ExitStack() as stack: + stack.enter_context(patch("rohmu.object_storage.google.get_credentials")) + stack.enter_context(patch("rohmu.object_storage.google.GoogleTransfer._create_object_store_if_needed_unwrapped")) + _init_google_client_mock = stack.enter_context( + patch("rohmu.object_storage.google.GoogleTransfer._init_google_client") + ) + + transfer = GoogleTransfer( + project_id="test-project-id", + bucket_name="test-bucket", + prefix="test-prefix/", + notifier=notifier, + ) + if preserve_trailing_slash is None: + transfer.delete_key(key) + else: + transfer.delete_key(key, preserve_trailing_slash=preserve_trailing_slash) + + mock_client_delete = _init_google_client_mock.return_value.objects().delete + mock_client_delete.assert_has_calls( + [ + call(bucket="test-bucket", object=expected_key), + call().execute(), + ] + ) + + +@pytest.mark.parametrize("preserve_trailing_slash", [True, False, None]) +def test_delete_keys(preserve_trailing_slash: Union[bool, None]) -> None: + notifier = MagicMock() + with ExitStack() as stack: + stack.enter_context(patch("rohmu.object_storage.google.get_credentials")) + stack.enter_context(patch("rohmu.object_storage.google.GoogleTransfer._create_object_store_if_needed_unwrapped")) + _init_google_client_mock = stack.enter_context( + patch("rohmu.object_storage.google.GoogleTransfer._init_google_client") + ) + + transfer = GoogleTransfer( + project_id="test-project-id", + bucket_name="test-bucket", + prefix="test-prefix/", + notifier=notifier, + ) + if preserve_trailing_slash is None: + transfer.delete_keys(["2", "3", "4/"]) + else: + transfer.delete_keys(["2", "3", "4/"], preserve_trailing_slash=preserve_trailing_slash) + + mock_client_delete = _init_google_client_mock.return_value.objects().delete + + mock_client_delete = _init_google_client_mock.return_value.objects().delete + expected_keys = ["2", "3", "4"] if not preserve_trailing_slash else ["2", "3", "4/"] + expected_calls = [] + for key in expected_keys: + expected_calls.extend( + [ + call(bucket="test-bucket", object=f"test-prefix/{key}"), + call().execute(), + ] + ) + mock_client_delete.assert_has_calls(expected_calls) diff --git a/test/object_storage/test_local.py b/test/object_storage/test_local.py index c690db2e..0f61fbb6 100644 --- a/test/object_storage/test_local.py +++ b/test/object_storage/test_local.py @@ -3,12 +3,14 @@ from functools import partial from io import BytesIO from itertools import cycle -from rohmu.errors import FileNotFoundFromStorageError, InvalidByteRangeError +from rohmu.errors import Error, FileNotFoundFromStorageError, InvalidByteRangeError from rohmu.object_storage.base import KEY_TYPE_OBJECT from rohmu.object_storage.local import LocalTransfer from tempfile import NamedTemporaryFile, TemporaryDirectory +from typing import Union from unittest.mock import MagicMock +import glob import hashlib import json import os @@ -297,3 +299,71 @@ def inc_progress(size: int) -> None: # we should not be able to find this with pytest.raises(FileNotFoundFromStorageError): transfer.get_metadata_for_key("test_key1") + + +@pytest.mark.parametrize( + ("key", "preserve_trailing_slash", "expected_key"), + [ + ("1", True, "test-prefix/1"), + ("2/", True, "test-prefix/2/"), + ("1", False, "test-prefix/1"), + ("2/", False, "test-prefix/2"), + ("1", None, "test-prefix/1"), + ("2/", None, "test-prefix/2"), + ], +) +def test_delete_key(key: str, preserve_trailing_slash: Union[bool, None], expected_key: str) -> None: + with TemporaryDirectory() as destdir: + notifier = MagicMock() + transfer = LocalTransfer( + directory=destdir, + notifier=notifier, + prefix="test-prefix/", + ) + + transfer.store_file_from_memory(key, memstring=b"Hello") + found_files = glob.glob(os.path.join(destdir, "test-prefix", "*")) + # ensure we have created some files + assert found_files + if preserve_trailing_slash: + with pytest.raises(Error): + transfer.delete_key(key, preserve_trailing_slash=True) + else: + if preserve_trailing_slash is None: + transfer.delete_key(key) + else: + transfer.delete_key(key, preserve_trailing_slash=preserve_trailing_slash) + + # all files got deleted + found_files = glob.glob(os.path.join(destdir, "test-prefix", "*")) + assert not found_files + + +@pytest.mark.parametrize("preserve_trailing_slash", [True, False, None]) +def test_delete_keys(preserve_trailing_slash: Union[bool, None]) -> None: + with TemporaryDirectory() as destdir: + notifier = MagicMock() + transfer = LocalTransfer( + directory=destdir, + notifier=notifier, + prefix="test-prefix/", + ) + + transfer.store_file_from_memory("2", memstring=b"Hello") + transfer.store_file_from_memory("3", memstring=b"Hello") + transfer.store_file_from_memory("4/", memstring=b"Hello") + found_files = glob.glob(os.path.join(destdir, "test-prefix", "*")) + # ensure we have created some files + assert found_files + if preserve_trailing_slash: + with pytest.raises(Error): + transfer.delete_keys(["2", "3", "4/"], preserve_trailing_slash=True) + else: + if preserve_trailing_slash is None: + transfer.delete_keys(["2", "3", "4/"]) + else: + transfer.delete_keys(["2", "3", "4/"], preserve_trailing_slash=preserve_trailing_slash) + + # all files got deleted + found_files = glob.glob(os.path.join(destdir, "test-prefix", "*")) + assert not found_files diff --git a/test/object_storage/test_s3.py b/test/object_storage/test_s3.py index e39d6fa2..c2185007 100644 --- a/test/object_storage/test_s3.py +++ b/test/object_storage/test_s3.py @@ -168,13 +168,41 @@ def test_operations_reporting(infra: S3Infra) -> None: infra.operation.assert_called_once_with(StorageOperation.head_request) -def test_deletion(infra: S3Infra) -> None: - infra.transfer.delete_keys(["2", "3"]) +@pytest.mark.parametrize("preserve_trailing_slash", [True, False, None]) +def test_delete_keys(infra: S3Infra, preserve_trailing_slash: Union[bool, None]) -> None: + if preserve_trailing_slash is None: + infra.transfer.delete_keys(["2", "3", "4/"]) + else: + infra.transfer.delete_keys(["2", "3", "4/"], preserve_trailing_slash=preserve_trailing_slash) infra.s3_client.delete_objects.assert_called_once_with( - Bucket="test-bucket", Delete={"Objects": [{"Key": "test-prefix/2"}, {"Key": "test-prefix/3"}]} + Bucket="test-bucket", + Delete={ + "Objects": [ + {"Key": "test-prefix/2"}, + {"Key": "test-prefix/3"}, + {"Key": "test-prefix/4/" if preserve_trailing_slash else "test-prefix/4"}, + ], + }, ) - infra.transfer.delete_key("1") - infra.s3_client.delete_object.assert_called_once_with(Bucket="test-bucket", Key="test-prefix/1") + + +@pytest.mark.parametrize( + ("key", "preserve_trailing_slash", "expected_key"), + [ + ("1", True, "test-prefix/1"), + ("2/", True, "test-prefix/2/"), + ("1", False, "test-prefix/1"), + ("2/", False, "test-prefix/2"), + ("1", None, "test-prefix/1"), + ("2/", None, "test-prefix/2"), + ], +) +def test_delete_key(infra: S3Infra, key: str, preserve_trailing_slash: Union[bool, None], expected_key: str) -> None: + if preserve_trailing_slash is None: + infra.transfer.delete_key(key) + else: + infra.transfer.delete_key(key, preserve_trailing_slash=preserve_trailing_slash) + infra.s3_client.delete_object.assert_called_once_with(Bucket="test-bucket", Key=expected_key) def test_get_contents_to_fileobj_raises_error_on_invalid_byte_range(infra: S3Infra) -> None: diff --git a/test/object_storage/test_sftp.py b/test/object_storage/test_sftp.py index d2e63d6c..6444f58d 100644 --- a/test/object_storage/test_sftp.py +++ b/test/object_storage/test_sftp.py @@ -1,10 +1,13 @@ # Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from datetime import datetime from io import BytesIO +from rohmu.errors import Error from rohmu.object_storage.sftp import SFTPTransfer from tempfile import NamedTemporaryFile -from typing import Any -from unittest.mock import MagicMock, patch +from typing import Any, Union +from unittest.mock import call, MagicMock, patch + +import pytest def test_store_file_from_disk() -> None: @@ -73,3 +76,77 @@ def upload_side_effect(*args: Any, **kwargs: Any) -> None: notifier.object_created.assert_called_once_with( key="test_key2", size=len(test_data), metadata={"Content-Length": "9", "some-date": "2022-11-15 18:30:58.486644"} ) + + +@pytest.mark.parametrize( + ("key", "preserve_trailing_slash", "expected_key"), + [ + ("1", True, ""), + ("2/", True, ""), + ("1", False, "test-prefix/1"), + ("2/", False, "test-prefix/2"), + ("1", None, "test-prefix/1"), + ("2/", None, "test-prefix/2"), + ], +) +def test_delete_key(key: str, preserve_trailing_slash: Union[bool, None], expected_key: str) -> None: + notifier = MagicMock() + with patch("paramiko.Transport") as _, patch("paramiko.SFTPClient") as sftp_client: + client = MagicMock() + sftp_client.from_transport.return_value = client + transfer = SFTPTransfer( + server="sftp.example.com", + port=2222, + username="testuser", + password="testpass", + notifier=notifier, + prefix="test-prefix/", + ) + + if preserve_trailing_slash: + with pytest.raises(Error): + transfer.delete_key(key, preserve_trailing_slash=True) + client.remove.assert_not_called() + else: + if preserve_trailing_slash is None: + transfer.delete_key(key) + else: + transfer.delete_key(key, preserve_trailing_slash=preserve_trailing_slash) + client.assert_has_calls( + [ + call.remove(expected_key.strip("/") + ".metadata"), + call.remove(expected_key.strip("/")), + ] + ) + + +@pytest.mark.parametrize("preserve_trailing_slash", [True, False, None]) +def test_delete_keys(preserve_trailing_slash: Union[bool, None]) -> None: + notifier = MagicMock() + with patch("paramiko.Transport") as _, patch("paramiko.SFTPClient") as sftp_client: + client = MagicMock() + sftp_client.from_transport.return_value = client + transfer = SFTPTransfer( + server="sftp.example.com", + port=2222, + username="testuser", + password="testpass", + notifier=notifier, + ) + test_data = b"test-data" + file_object = BytesIO(test_data) + + # Size reporting relies on the progress callback from paramiko + def upload_side_effect(*args: Any, **kwargs: Any) -> None: + if kwargs.get("callback"): + kwargs["callback"](len(test_data), len(test_data)) + + client.putfo = MagicMock(wraps=upload_side_effect) + + metadata = {"Content-Length": len(test_data), "some-date": datetime(2022, 11, 15, 18, 30, 58, 486644)} + transfer.store_file_object(key="test_key2", fd=file_object, metadata=metadata) + + client.putfo.assert_called() + notifier.object_created.assert_called_once_with( + key="test_key2", size=len(test_data), metadata={"Content-Length": "9", "some-date": "2022-11-15 18:30:58.486644"} + ) diff --git a/test/object_storage/test_swift.py b/test/object_storage/test_swift.py index 91fc142f..39c07d9f 100644 --- a/test/object_storage/test_swift.py +++ b/test/object_storage/test_swift.py @@ -3,7 +3,8 @@ from io import BytesIO from tempfile import NamedTemporaryFile from types import ModuleType -from unittest.mock import MagicMock, patch +from typing import Union +from unittest.mock import call, MagicMock, patch import pytest import sys @@ -76,3 +77,73 @@ def test_iter_key_with_empty_key(swift_module: ModuleType) -> None: ) list(transfer.iter_key("")) transfer.conn.get_container.assert_called_with("test_container", prefix="", full_listing=True, delimiter="/") + + +@pytest.mark.parametrize( + ("key", "preserve_trailing_slash", "expected_key"), + [ + ("1", True, "test-prefix/1"), + ("2/", True, "test-prefix/2/"), + ("1", False, "test-prefix/1"), + ("2/", False, "test-prefix/2"), + ("1", None, "test-prefix/1"), + ("2/", None, "test-prefix/2"), + ], +) +def test_delete_key(swift_module: ModuleType, key: str, preserve_trailing_slash: Union[bool, None], expected_key: str) -> None: + notifier = MagicMock() + connection = MagicMock() + swift_module.client.Connection.return_value = connection + transfer = swift_module.SwiftTransfer( + user="testuser", + key="testkey", + container_name="test_container", + auth_url="http://auth.example.com", + notifier=notifier, + prefix="test-prefix/", + ) + if preserve_trailing_slash is None: + transfer.delete_key(key=key) + else: + transfer.delete_key(key=key, preserve_trailing_slash=preserve_trailing_slash) + + connection.assert_has_calls( + [ + # ensure container exists + call.get_container("test_container", headers={}, limit=1), + call.head_object("test_container", expected_key), + call.head_object().__contains__("x-object-manifest"), + call.delete_object("test_container", expected_key), + ] + ) + + +@pytest.mark.parametrize("preserve_trailing_slash", [True, False, None]) +def test_delete_keys(swift_module: ModuleType, preserve_trailing_slash: Union[bool, None]) -> None: + notifier = MagicMock() + connection = MagicMock() + swift_module.client.Connection.return_value = connection + transfer = swift_module.SwiftTransfer( + user="testuser", + key="testkey", + container_name="test_container", + auth_url="http://auth.example.com", + notifier=notifier, + prefix="test-prefix/", + ) + if preserve_trailing_slash is None: + transfer.delete_keys(["2", "3", "4/"]) + else: + transfer.delete_keys(["2", "3", "4/"], preserve_trailing_slash=preserve_trailing_slash) + + expected_calls = [call.get_container("test_container", headers={}, limit=1)] + expected_keys = ["2", "3", "4/"] if preserve_trailing_slash else ["2", "3", "4"] + for expected_key in expected_keys: + expected_calls.extend( + [ + call.head_object("test_container", f"test-prefix/{expected_key}"), + call.head_object().__contains__("x-object-manifest"), + call.delete_object("test_container", f"test-prefix/{expected_key}"), + ] + ) + connection.assert_has_calls(expected_calls)