Better error reporting in logs

tilecloud/filter/logger.py

@@ -19,3 +19,9 @@ def __call__(self, tile: Optional[Tile]) -> Optional[Tile]:
             variables.update(tile.tilecoord.__dict__)
             self.logger.log(self.level, self.msgformat, variables, *self.args, **self.kwargs)
         return tile
+
+    def __str__(self) -> str:
+        return f"{self.logger.name}({self.level},  {self.msgformat})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

tilecloud/store/azure_storage_blob.py

@@ -67,7 +67,7 @@ def delete_one(self, tile: Tile) -> Tile:
                 if blob.exists():
                     blob.delete_blob()
         except Exception as exc:  # pylint: disable=broad-except
-            _LOGGER.exception(exc)
+            _LOGGER.warning("Failed to delete tile %s", tile.tilecoord, exc_info=exc)
             tile.error = exc
         return tile
 
@@ -84,7 +84,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]:
             tile.content_encoding = properties.content_settings.content_encoding
             tile.content_type = properties.content_settings.content_type
         except Exception as exc:  # pylint: disable=broad-except
-            _LOGGER.exception(exc)
+            _LOGGER.warning("Failed to get tile %s", tile.tilecoord, exc_info=exc)
             tile.error = exc
         return tile
 
@@ -115,7 +115,7 @@ def put_one(self, tile: Tile) -> Tile:
                     ),
                 )
             except Exception as exc:  # pylint: disable=broad-except
-                _LOGGER.exception(exc)
+                _LOGGER.warning("Failed to put tile %s", tile.tilecoord, exc_info=exc)
                 tile.error = exc
 
         return tile

tilecloud/store/filesystem.py

@@ -1,11 +1,14 @@
 import errno
+import logging
 import os
 import os.path
 from collections.abc import Iterator
 from typing import Any, Optional
 
 from tilecloud import Tile, TileLayout, TileStore
 
+_LOGGER = logging.getLogger(__name__)
+
 
 class FilesystemTileStore(TileStore):
     """
@@ -21,6 +24,7 @@ def delete_one(self, tile: Tile) -> Tile:
         try:
             filename = self.tilelayout.filename(tile.tilecoord, tile.metadata)
         except Exception as exception:  # pylint: disable=broad-except
+            _LOGGER.warning("Error while deleting tile %s", tile, exc_info=True)
             tile.error = exception
             return tile
         if os.path.exists(filename):
@@ -37,6 +41,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]:
         try:
             filename = self.tilelayout.filename(tile.tilecoord, tile.metadata)
         except Exception as exception:  # pylint: disable=broad-except
+            _LOGGER.warning("Error while getting tile %s", tile, exc_info=True)
             tile.error = exception
             return tile
         try:
@@ -64,6 +69,7 @@ def put_one(self, tile: Tile) -> Tile:
         try:
             filename = self.tilelayout.filename(tile.tilecoord, tile.metadata)
         except Exception as exception:  # pylint: disable=broad-except
+            _LOGGER.warning("Error while putting tile %s", tile, exc_info=True)
             tile.error = exception
             return tile
         dirname = os.path.dirname(filename)

tilecloud/store/s3.py

@@ -10,9 +10,9 @@
 
 from tilecloud import Tile, TileLayout, TileStore
 
-logger = logging.getLogger(__name__)
-lock = threading.Lock()
-CLIENT_TIMEOUT = 60
+_LOGGER = logging.getLogger(__name__)
+_LOCK = threading.Lock()
+_CLIENT_TIMEOUT = 60
 
 
 class S3TileStore(TileStore):
@@ -55,6 +55,7 @@ def delete_one(self, tile: Tile) -> Tile:
             if not self.dry_run:
                 self.client.delete_object(Bucket=self.bucket, Key=key_name)
         except botocore.exceptions.ClientError as exc:
+            _LOGGER.warning("Error while deleting tile %s", tile, exc_info=True)
             tile.error = exc
         return tile
 
@@ -68,6 +69,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]:
         except botocore.exceptions.ClientError as exc:
             if _get_status(exc) == 404:
                 return None
+            _LOGGER.error("Error while getting tile %s", tile, exc_info=True)
             tile.error = exc
         return tile
 
@@ -96,6 +98,7 @@ def put_one(self, tile: Tile) -> Tile:
                     ACL="public-read", Body=tile.data, Key=key_name, Bucket=self.bucket, **args
                 )
             except botocore.exceptions.ClientError as exc:
+                _LOGGER.warning("Error while putting tile %s", tile, exc_info=True)
                 tile.error = exc
         return tile
 
@@ -111,8 +114,8 @@ def _get_status(s3_client_exception: botocore.exceptions.ClientError) -> int:
 
 
 def get_client(s3_host: Optional[str]) -> "botocore.client.S3":
-    config = botocore.config.Config(connect_timeout=CLIENT_TIMEOUT, read_timeout=CLIENT_TIMEOUT)
-    with lock:
+    config = botocore.config.Config(connect_timeout=_CLIENT_TIMEOUT, read_timeout=_CLIENT_TIMEOUT)
+    with _LOCK:
         return boto3.client(
             "s3", endpoint_url=(f"https://{s3_host}/") if s3_host is not None else None, config=config
         )

tilecloud/store/sqs.py

@@ -10,15 +10,15 @@
 from tilecloud import Tile, TileStore
 from tilecloud.store.queue import decode_message, encode_message
 
-BATCH_SIZE = 10  # max Amazon allows
-logger = logging.getLogger(__name__)
+_BATCH_SIZE = 10  # max Amazon allows
+_LOGGER = logging.getLogger(__name__)
 
 
 def maybe_stop(queue: "botocore.client.SQS") -> bool:
     try:
         queue.load()
     except botocore.exceptions.EndpointConnectionError:
-        logger.warning("Error fetching SQS attributes", exc_info=True)
+        _LOGGER.warning("Error fetching SQS attributes", exc_info=True)
         return True
 
     attributes = queue.attributes
@@ -49,9 +49,9 @@ def get_one(self, tile: Tile) -> Tile:
     def list(self) -> Iterator[Tile]:
         while True:
             try:
-                sqs_messages = self.queue.receive_messages(MaxNumberOfMessages=BATCH_SIZE)
+                sqs_messages = self.queue.receive_messages(MaxNumberOfMessages=_BATCH_SIZE)
             except botocore.exceptions.EndpointConnectionError:
-                logger.warning("Error fetching SQS messages", exc_info=True)
+                _LOGGER.warning("Error fetching SQS messages", exc_info=True)
                 sqs_messages = []
 
             if not sqs_messages:
@@ -63,7 +63,7 @@ def list(self) -> Iterator[Tile]:
                         tile = decode_message(sqs_message.body.encode("utf-8"), sqs_message=sqs_message)
                         yield tile
                     except Exception:  # pylint: disable=broad-except
-                        logger.warning("Failed decoding the SQS message", exc_info=True)
+                        _LOGGER.warning("Failed decoding the SQS message", exc_info=True)
                         sqs_message.delete()
 
     def delete_one(self, tile: Tile) -> Tile:
@@ -78,7 +78,7 @@ def put_one(self, tile: Tile) -> Tile:
         try:
             self.queue.send_message(MessageBody=sqs_message)
         except Exception as exception:  # pylint: disable=broad-except
-            logger.warning("Failed sending SQS message", exc_info=True)
+            _LOGGER.warning("Failed sending SQS message", exc_info=True)
             tile.error = exception
         return tile
 
@@ -87,7 +87,7 @@ def put(self, tiles: Iterable[Tile]) -> Iterator[Tile]:
         try:
             for tile in tiles:
                 buffered_tiles.append(tile)
-                if len(buffered_tiles) >= BATCH_SIZE:
+                if len(buffered_tiles) >= _BATCH_SIZE:
                     self._send_buffer(buffered_tiles)
                     buffered_tiles = []
                 yield tile
@@ -102,11 +102,11 @@ def _send_buffer(self, tiles: builtins.list[Tile]) -> None:
             ]
             response = self.queue.send_messages(Entries=messages)
             for failed in response.get("Failed", []):
-                logger.warning("Failed sending SQS message: %s", failed["Message"])
+                _LOGGER.warning("Failed sending SQS message: %s", failed["Message"])
                 pos = int(failed["Id"])
                 tiles[pos].error = failed["Message"]
         except Exception as exception:  # pylint: disable=broad-except
-            logger.warning("Failed sending SQS messages", exc_info=True)
+            _LOGGER.warning("Failed sending SQS messages", exc_info=True)
             for tile in tiles:
                 tile.error = exception
 

tilecloud/store/url.py

@@ -6,7 +6,7 @@
 
 from tilecloud import NotSupportedOperation, Tile, TileLayout, TileStore
 
-logger = logging.getLogger(__name__)
+_LOGGER = logging.getLogger(__name__)
 
 
 class URLTileStore(TileStore):
@@ -34,14 +34,15 @@ def get_one(self, tile: Tile) -> Optional[Tile]:
         try:
             url = tilelayout.filename(tile.tilecoord, tile.metadata)
         except Exception as exception:  # pylint: disable=broad-except
+            _LOGGER.warning("Error while getting tile %s", tile, exc_info=True)
             tile.error = exception
             return tile
 
-        logger.info("GET %s", url)
+        _LOGGER.info("GET %s", url)
         try:
             response = self.session.get(url)
             if response.status_code in (404, 204):
-                logger.debug("Got empty tile from %s: %s", url, response.status_code)
+                _LOGGER.debug("Got empty tile from %s: %s", url, response.status_code)
                 return None
             tile.content_encoding = response.headers.get("Content-Encoding")
             tile.content_type = response.headers.get("Content-Type")
@@ -64,6 +65,7 @@ def get_one(self, tile: Tile) -> Optional[Tile]:
             else:
                 tile.error = f"URL: {url}\n{response.status_code}: {response.reason}\n{response.text}"
         except requests.exceptions.RequestException as exception:
+            _LOGGER.warning("Error while getting tile %s", tile, exc_info=True)
             tile.error = exception
         return tile