diff --git a/tilecloud/filter/logger.py b/tilecloud/filter/logger.py index 69d8b9fd5..8be615233 100644 --- a/tilecloud/filter/logger.py +++ b/tilecloud/filter/logger.py @@ -19,3 +19,9 @@ def __call__(self, tile: Optional[Tile]) -> Optional[Tile]: variables.update(tile.tilecoord.__dict__) self.logger.log(self.level, self.msgformat, variables, *self.args, **self.kwargs) return tile + + def __str__(self) -> str: + return f"{self.logger.name}({self.level}, {self.msgformat})" + + def __repr__(self) -> str: + return self.__str__() diff --git a/tilecloud/store/azure_storage_blob.py b/tilecloud/store/azure_storage_blob.py index 43e4054b4..cda9c6a77 100644 --- a/tilecloud/store/azure_storage_blob.py +++ b/tilecloud/store/azure_storage_blob.py @@ -67,7 +67,7 @@ def delete_one(self, tile: Tile) -> Tile: if blob.exists(): blob.delete_blob() except Exception as exc: # pylint: disable=broad-except - _LOGGER.exception(exc) + _LOGGER.warning("Failed to delete tile %s", tile.tilecoord, exc_info=exc) tile.error = exc return tile @@ -84,7 +84,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]: tile.content_encoding = properties.content_settings.content_encoding tile.content_type = properties.content_settings.content_type except Exception as exc: # pylint: disable=broad-except - _LOGGER.exception(exc) + _LOGGER.warning("Failed to get tile %s", tile.tilecoord, exc_info=exc) tile.error = exc return tile @@ -115,7 +115,7 @@ def put_one(self, tile: Tile) -> Tile: ), ) except Exception as exc: # pylint: disable=broad-except - _LOGGER.exception(exc) + _LOGGER.warning("Failed to put tile %s", tile.tilecoord, exc_info=exc) tile.error = exc return tile diff --git a/tilecloud/store/filesystem.py b/tilecloud/store/filesystem.py index 1591f91b4..5bb6b711c 100644 --- a/tilecloud/store/filesystem.py +++ b/tilecloud/store/filesystem.py @@ -1,4 +1,5 @@ import errno +import logging import os import os.path from collections.abc import Iterator @@ -6,6 +7,8 @@ from tilecloud import Tile, TileLayout, TileStore +_LOGGER = logging.getLogger(__name__) + class FilesystemTileStore(TileStore): """ @@ -21,6 +24,7 @@ def delete_one(self, tile: Tile) -> Tile: try: filename = self.tilelayout.filename(tile.tilecoord, tile.metadata) except Exception as exception: # pylint: disable=broad-except + _LOGGER.warning("Error while deleting tile %s", tile, exc_info=True) tile.error = exception return tile if os.path.exists(filename): @@ -37,6 +41,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]: try: filename = self.tilelayout.filename(tile.tilecoord, tile.metadata) except Exception as exception: # pylint: disable=broad-except + _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) tile.error = exception return tile try: @@ -64,6 +69,7 @@ def put_one(self, tile: Tile) -> Tile: try: filename = self.tilelayout.filename(tile.tilecoord, tile.metadata) except Exception as exception: # pylint: disable=broad-except + _LOGGER.warning("Error while putting tile %s", tile, exc_info=True) tile.error = exception return tile dirname = os.path.dirname(filename) diff --git a/tilecloud/store/s3.py b/tilecloud/store/s3.py index ef37fee1f..efaf478b2 100644 --- a/tilecloud/store/s3.py +++ b/tilecloud/store/s3.py @@ -10,9 +10,9 @@ from tilecloud import Tile, TileLayout, TileStore -logger = logging.getLogger(__name__) -lock = threading.Lock() -CLIENT_TIMEOUT = 60 +_LOGGER = logging.getLogger(__name__) +_LOCK = threading.Lock() +_CLIENT_TIMEOUT = 60 class S3TileStore(TileStore): @@ -55,6 +55,7 @@ def delete_one(self, tile: Tile) -> Tile: if not self.dry_run: self.client.delete_object(Bucket=self.bucket, Key=key_name) except botocore.exceptions.ClientError as exc: + _LOGGER.warning("Error while deleting tile %s", tile, exc_info=True) tile.error = exc return tile @@ -68,6 +69,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]: except botocore.exceptions.ClientError as exc: if _get_status(exc) == 404: return None + _LOGGER.error("Error while getting tile %s", tile, exc_info=True) tile.error = exc return tile @@ -96,6 +98,7 @@ def put_one(self, tile: Tile) -> Tile: ACL="public-read", Body=tile.data, Key=key_name, Bucket=self.bucket, **args ) except botocore.exceptions.ClientError as exc: + _LOGGER.warning("Error while putting tile %s", tile, exc_info=True) tile.error = exc return tile @@ -111,8 +114,8 @@ def _get_status(s3_client_exception: botocore.exceptions.ClientError) -> int: def get_client(s3_host: Optional[str]) -> "botocore.client.S3": - config = botocore.config.Config(connect_timeout=CLIENT_TIMEOUT, read_timeout=CLIENT_TIMEOUT) - with lock: + config = botocore.config.Config(connect_timeout=_CLIENT_TIMEOUT, read_timeout=_CLIENT_TIMEOUT) + with _LOCK: return boto3.client( "s3", endpoint_url=(f"https://{s3_host}/") if s3_host is not None else None, config=config ) diff --git a/tilecloud/store/sqs.py b/tilecloud/store/sqs.py index ff2835851..a8d8f5e38 100644 --- a/tilecloud/store/sqs.py +++ b/tilecloud/store/sqs.py @@ -10,15 +10,15 @@ from tilecloud import Tile, TileStore from tilecloud.store.queue import decode_message, encode_message -BATCH_SIZE = 10 # max Amazon allows -logger = logging.getLogger(__name__) +_BATCH_SIZE = 10 # max Amazon allows +_LOGGER = logging.getLogger(__name__) def maybe_stop(queue: "botocore.client.SQS") -> bool: try: queue.load() except botocore.exceptions.EndpointConnectionError: - logger.warning("Error fetching SQS attributes", exc_info=True) + _LOGGER.warning("Error fetching SQS attributes", exc_info=True) return True attributes = queue.attributes @@ -49,9 +49,9 @@ def get_one(self, tile: Tile) -> Tile: def list(self) -> Iterator[Tile]: while True: try: - sqs_messages = self.queue.receive_messages(MaxNumberOfMessages=BATCH_SIZE) + sqs_messages = self.queue.receive_messages(MaxNumberOfMessages=_BATCH_SIZE) except botocore.exceptions.EndpointConnectionError: - logger.warning("Error fetching SQS messages", exc_info=True) + _LOGGER.warning("Error fetching SQS messages", exc_info=True) sqs_messages = [] if not sqs_messages: @@ -63,7 +63,7 @@ def list(self) -> Iterator[Tile]: tile = decode_message(sqs_message.body.encode("utf-8"), sqs_message=sqs_message) yield tile except Exception: # pylint: disable=broad-except - logger.warning("Failed decoding the SQS message", exc_info=True) + _LOGGER.warning("Failed decoding the SQS message", exc_info=True) sqs_message.delete() def delete_one(self, tile: Tile) -> Tile: @@ -78,7 +78,7 @@ def put_one(self, tile: Tile) -> Tile: try: self.queue.send_message(MessageBody=sqs_message) except Exception as exception: # pylint: disable=broad-except - logger.warning("Failed sending SQS message", exc_info=True) + _LOGGER.warning("Failed sending SQS message", exc_info=True) tile.error = exception return tile @@ -87,7 +87,7 @@ def put(self, tiles: Iterable[Tile]) -> Iterator[Tile]: try: for tile in tiles: buffered_tiles.append(tile) - if len(buffered_tiles) >= BATCH_SIZE: + if len(buffered_tiles) >= _BATCH_SIZE: self._send_buffer(buffered_tiles) buffered_tiles = [] yield tile @@ -102,11 +102,11 @@ def _send_buffer(self, tiles: builtins.list[Tile]) -> None: ] response = self.queue.send_messages(Entries=messages) for failed in response.get("Failed", []): - logger.warning("Failed sending SQS message: %s", failed["Message"]) + _LOGGER.warning("Failed sending SQS message: %s", failed["Message"]) pos = int(failed["Id"]) tiles[pos].error = failed["Message"] except Exception as exception: # pylint: disable=broad-except - logger.warning("Failed sending SQS messages", exc_info=True) + _LOGGER.warning("Failed sending SQS messages", exc_info=True) for tile in tiles: tile.error = exception diff --git a/tilecloud/store/url.py b/tilecloud/store/url.py index e868006ab..7f3e52fdb 100644 --- a/tilecloud/store/url.py +++ b/tilecloud/store/url.py @@ -6,7 +6,7 @@ from tilecloud import NotSupportedOperation, Tile, TileLayout, TileStore -logger = logging.getLogger(__name__) +_LOGGER = logging.getLogger(__name__) class URLTileStore(TileStore): @@ -34,14 +34,15 @@ def get_one(self, tile: Tile) -> Optional[Tile]: try: url = tilelayout.filename(tile.tilecoord, tile.metadata) except Exception as exception: # pylint: disable=broad-except + _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) tile.error = exception return tile - logger.info("GET %s", url) + _LOGGER.info("GET %s", url) try: response = self.session.get(url) if response.status_code in (404, 204): - logger.debug("Got empty tile from %s: %s", url, response.status_code) + _LOGGER.debug("Got empty tile from %s: %s", url, response.status_code) return None tile.content_encoding = response.headers.get("Content-Encoding") tile.content_type = response.headers.get("Content-Type") @@ -64,6 +65,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]: else: tile.error = f"URL: {url}\n{response.status_code}: {response.reason}\n{response.text}" except requests.exceptions.RequestException as exception: + _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) tile.error = exception return tile