diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index 728bda535..05c3cc7a8 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -208,7 +208,7 @@ class Run: def __init__( self, gene: "TileGeneration", - functions: list[Callable[[Tile], Tile]], + functions: list[Callable[[Tile], Tile | None]], out: IO[str] | None = None, ) -> None: self.gene = gene @@ -250,6 +250,7 @@ def __call__(self, tile: Tile | None) -> Tile | None: tile = func(tile) except Exception as e: # pylint: disable=broad-exception-caught _LOGGER.exception("[%s] Fail to process function %s", tilecoord, func) + assert tile is not None tile.error = e else: tile = func(tile) @@ -447,9 +448,9 @@ def __init__( self._close_actions: list[Close] = [] self.error_lock = threading.Lock() self.error_files_: dict[str, TextIO] = {} - self.functions_tiles: list[Callable[[Tile], Tile]] = [] - self.functions_metatiles: list[Callable[[Tile], Tile]] = [] - self.functions = self.functions_metatiles + self.functions_tiles: list[Callable[[Tile], Tile | None]] = [] + self.functions_metatiles: list[Callable[[Tile], Tile | None]] = [] + self.functions: list[Callable[[Tile], Tile | None]] = self.functions_metatiles self.metatilesplitter_thread_pool: ThreadPoolExecutor | None = None self.multi_thread = multi_thread self.maxconsecutive_errors = maxconsecutive_errors @@ -1367,16 +1368,19 @@ def imap(self, func: Any, time_message: str | None = None) -> None: class Func: """Function with an additional field used to names it in timing messages.""" - def __init__(self, func: Callable[[Tile], Tile], time_message: str | None) -> None: + def __init__(self, func: Callable[[Tile], Tile | None], time_message: str | None) -> None: self.func = func self.time_message = time_message - def __call__(self, tile: Tile) -> Tile: + def __call__(self, tile: Tile) -> Tile | None: return self.func(tile) def __str__(self) -> str: return f"Func: {self.func}" + def __repr__(self) -> str: + return f"Func: {self.func!r}" + self.functions.append(Func(func, time_message)) def consume(self, test: int | None = None) -> None: diff --git a/tilecloud_chain/multitilestore.py b/tilecloud_chain/multitilestore.py index cef73bc61..e237d6ffd 100644 --- a/tilecloud_chain/multitilestore.py +++ b/tilecloud_chain/multitilestore.py @@ -25,6 +25,12 @@ def _get_store(self, config_file: str, layer: str) -> TileStore | None: self.stores[(config_file, layer)] = store return store + def _get_store_tile(self, tile: Tile) -> TileStore | None: + """Return the store corresponding to the tile.""" + layer = tile.metadata["layer"] + config_file = tile.metadata["config_file"] + return self._get_store(config_file, layer) + def __contains__(self, tile: Tile) -> bool: """ Return true if this store contains ``tile``. @@ -32,9 +38,7 @@ def __contains__(self, tile: Tile) -> bool: Arguments: tile: Tile """ - layer = tile.metadata["layer"] - config_file = tile.metadata["config_file"] - store = self._get_store(config_file, layer) + store = self._get_store_tile(tile) assert store is not None return tile in store @@ -45,9 +49,7 @@ def delete_one(self, tile: Tile) -> Tile: Arguments: tile: Tile """ - layer = tile.metadata["layer"] - config_file = tile.metadata["config_file"] - store = self._get_store(config_file, layer) + store = self._get_store_tile(tile) assert store is not None return store.delete_one(tile) @@ -64,9 +66,7 @@ def put_one(self, tile: Tile) -> Tile: Arguments: tile: Tile """ - layer = tile.metadata["layer"] - config_file = tile.metadata["config_file"] - store = self._get_store(config_file, layer) + store = self._get_store_tile(tile) assert store is not None return store.put_one(tile) @@ -77,9 +77,7 @@ def get_one(self, tile: Tile) -> Tile | None: Arguments: tile: Tile """ - layer = tile.metadata["layer"] - config_file = tile.metadata["config_file"] - store = self._get_store(config_file, layer) + store = self._get_store_tile(tile) assert store is not None return store.get_one(tile) @@ -117,13 +115,13 @@ def apply(key: tuple[str, str], tiles: Iterator[Tile]) -> Iterator[Tile]: def __str__(self) -> str: """Return a string representation of the object.""" stores = {str(store) for store in self.stores.values()} - keys = {f"{config_file}/{layer}" for config_file, layer in self.stores} + keys = {f"{config_file}:{layer}" for config_file, layer in self.stores} return f"{self.__class__.__name__}({', '.join(stores)} - {', '.join(keys)})" def __repr__(self) -> str: """Return a string representation of the object.""" stores = {repr(store) for store in self.stores.values()} - keys = {f"{config_file}/{layer}" for config_file, layer in self.stores} + keys = {f"{config_file}:{layer}" for config_file, layer in self.stores} return f"{self.__class__.__name__}({', '.join(stores)} - {', '.join(keys)})" @staticmethod