diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index 974c67a93..311a713eb 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -26,7 +26,7 @@ from io import BytesIO from itertools import product from math import ceil, sqrt -from typing import IO, TYPE_CHECKING, Any, TextIO, TypedDict, cast +from typing import IO, TYPE_CHECKING, Any, Literal, TextIO, TypedDict, cast import boto3 import botocore.client @@ -833,8 +833,11 @@ def get_store( cache: tilecloud_chain.configuration.Cache, layer_name: str, read_only: bool = False, - ) -> TileStore: + ) -> TileStore | None: """Get the tile store.""" + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file) + return None layer = config.config["layers"][layer_name] grid = config.config["grids"][layer["grid"]] layout = WMTSTileLayout( @@ -928,7 +931,7 @@ def get_tilesstore(self, cache: str | None = None) -> TimedTileStoreWrapper: """Get the tile store.""" gene = self - def get_store(config_file: str, layer_name: str) -> TileStore: + def get_store(config_file: str, layer_name: str) -> TileStore | None: config = gene.get_config(config_file) cache_name = cache or config.config["generation"].get( "default_cache", configuration.DEFAULT_CACHE_DEFAULT @@ -972,6 +975,9 @@ def add_metatile_splitter(self, store: TileStore | None = None) -> None: def get_splitter(config_file: str, layer_name: str) -> MetaTileSplitterTileStore | None: config = gene.get_config(config_file) + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config_file) + return None layer = config.config["layers"][layer_name] if layer.get("meta"): return MetaTileSplitterTileStore( @@ -1098,6 +1104,9 @@ def get_geoms( if dated_geoms is not None and config.mtime == dated_geoms.mtime: return dated_geoms.geoms + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file) + return {} layer = config.config["layers"][layer_name] if self.options.near is not None or ( @@ -1187,6 +1196,9 @@ def get_geoms( def init_tilecoords(self, config: DatedConfig, layer_name: str) -> None: """Initialize the tilestream for the given layer.""" + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file) + return layer = config.config["layers"][layer_name] resolutions = config.config["grids"][layer["grid"]]["resolutions"] @@ -1296,6 +1308,9 @@ def _tilestream( def set_tilecoords(self, config: DatedConfig, tilecoords: Iterable[TileCoord], layer_name: str) -> None: """Set the tilestream for the given tilecoords.""" assert tilecoords is not None + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file) + return layer = config.config["layers"][layer_name] metadata = {"layer": layer_name, "config_file": config.file} @@ -1327,6 +1342,9 @@ def process(self, name: str | None = None, key: str = "post_process") -> None: def get_process(config_file: str, layer_name: str) -> Process | None: config = gene.get_config(config_file) + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config_file) + return None layer = config.config["layers"][layer_name] name_ = name if name_ is None: @@ -1575,7 +1593,7 @@ def __init__( get_action: Callable[[str, str], Callable[[Tile], Tile | None] | None], ) -> None: self.get_action = get_action - self.actions: dict[tuple[str, str], _DatedAction | None] = {} + self.actions: dict[tuple[str, str], _DatedAction] = {} def _get_action(self, config_file: str, layer: str) -> Callable[[Tile], Tile | None] | None: """Get the action based on the tile's layer name.""" @@ -1606,13 +1624,13 @@ def __call__(self, tile: Tile) -> Tile | None: def __str__(self) -> str: """Return a string representation of the object.""" - actions = {str(action) for action in self.actions.values()} + actions = {str(action.action) for action in self.actions.values()} keys = {f"{config_file}:{layer}" for config_file, layer in self.actions} return f"{self.__class__.__name__}({', '.join(actions)} - {', '.join(keys)})" def __repr__(self) -> str: """Return a string representation of the object.""" - actions = {repr(action) for action in self.actions.values()} + actions = {repr(action.action) for action in self.actions.values()} keys = {f"{config_file}:{layer}" for config_file, layer in self.actions} return f"{self.__class__.__name__}({', '.join(actions)} - {', '.join(keys)})" @@ -1689,6 +1707,9 @@ def filter_tilecoord( self, config: DatedConfig, tilecoord: TileCoord, layer_name: str, host: str | None = None ) -> bool: """Filter the tilecoord.""" + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file) + return True layer = config.config["layers"][layer_name] grid_name = layer["grid"] grid = config.config["grids"][grid_name] @@ -1780,7 +1801,15 @@ class Process: def __init__(self, config: tilecloud_chain.configuration.ProcessCommand, options: Namespace) -> None: self.config = config - self.options = options + self.options: list[Literal["verbose"] | Literal["debug"] | Literal["quiet"] | Literal["default"]] = [] + if options.verbose: + self.options.append("verbose") + if options.debug: + self.options.append("debug") + if options.quiet: + self.options.append("quiet") + if not self.options: + self.options.append("default") def __call__(self, tile: Tile) -> Tile | None: """Process the tile.""" @@ -1791,16 +1820,9 @@ def __call__(self, tile: Tile) -> Tile | None: for cmd in self.config: args = [] - if ( - not self.options.verbose and not self.options.debug and not self.options.quiet - ) and "default" in cmd["arg"]: - args.append(cmd["arg"]["default"]) - if self.options.verbose and "verbose" in cmd["arg"]: - args.append(cmd["arg"]["verbose"]) - if self.options.debug and "debug" in cmd["arg"]: - args.append(cmd["arg"]["debug"]) - if self.options.quiet and "quiet" in cmd["arg"]: - args.append(cmd["arg"]["quiet"]) + for option in self.options: + if option in cmd["arg"]: + args.append(cmd["arg"][option]) if cmd.get("need_out", configuration.NEED_OUT_DEFAULT): fd_out, name_out = tempfile.mkstemp() @@ -1843,6 +1865,12 @@ def __call__(self, tile: Tile) -> Tile | None: return tile + def __str__(self) -> str: + return f"{self.__class__.__name__}({', '.join(self.options)} - {self.config})" + + def __repr__(self) -> str: + return self.__str__() + class TilesFileStore(TileStore): """Load tiles to be generate from a file.""" diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py index d354658e5..e2349625b 100644 --- a/tilecloud_chain/controller.py +++ b/tilecloud_chain/controller.py @@ -216,7 +216,7 @@ def get_wmts_capabilities( str, jinja2_template( data.decode("utf-8"), - layers=config.config["layers"], + layers=config.config.get("layers", {}), layer_legends=gene.layer_legends, grids=config.config["grids"], getcapabilities=urljoin( # type: ignore @@ -304,7 +304,7 @@ def _fill_legend( with concurrent.futures.ThreadPoolExecutor( max_workers=int(os.environ.get("TILECLOUD_CHAIN_CONCURRENT_GET_LEGEND", "10")) ) as executor: - for layer_name, layer in config.config["layers"].items(): + for layer_name, layer in config.config.get("layers", {}).items(): if ( "legend_mime" in layer and "legend_extension" in layer @@ -341,7 +341,7 @@ def _fill_legend( _LOGGER.debug("Get %i legend images in %s", len(legend_image_future), time.perf_counter() - start) - for layer_name, layer in config.config["layers"].items(): + for layer_name, layer in config.config.get("layers", {}).items(): previous_legend: tilecloud_chain.Legend | None = None previous_resolution = None if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends: @@ -472,14 +472,14 @@ def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) -> config = gene.get_config(gene.config_file) cache = config.config["caches"][gene.options.cache] - for layer_name, layer in config.config["layers"].items(): + for layer_name, layer in config.config.get("layers", {}).items(): if "legend_mime" in layer and "legend_extension" in layer and layer["type"] == "wms": session = requests.session() session.headers.update(layer["headers"]) previous_hash = None for zoom, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]): legends = [] - for wms_layer in layer["layers"].split(","): + for wms_layer in layer.get("layers", "").split(","): url = ( layer["url"] + "?" diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py index 050fcab9f..fde09f077 100644 --- a/tilecloud_chain/generate.py +++ b/tilecloud_chain/generate.py @@ -195,6 +195,11 @@ def _generate_queue(self, layer_name: str | None) -> None: assert layer_name is not None assert self._gene.config_file is not None config = self._gene.get_config(self._gene.config_file) + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning( + "Layer '%s' not found in the configuration file '%s'", layer_name, self._gene.config_file + ) + sys.exit(1) layer = config.config["layers"][layer_name] if self._options.get_bbox: @@ -215,6 +220,11 @@ def _generate_queue(self, layer_name: str | None) -> None: self._gene.init_tilecoords(config, layer_name) elif self._options.role == "hash": + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning( + "Layer '%s' not found in the configuration file '%s'", layer_name, self._gene.config_file + ) + sys.exit(1) layer = config.config["layers"][layer_name] try: z, x, y = (int(v) for v in self._options.get_hash.split("/")) @@ -371,6 +381,13 @@ def generate_resume(self, layer_name: str | None) -> None: layer = None if layer_name is not None: assert config is not None + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning( + "Layer '%s' not found in the configuration file '%s'", + layer_name, + self._gene.config_file, + ) + sys.exit(1) layer = config.config["layers"][layer_name] all_dimensions = self._gene.get_all_dimensions(layer) formatted_dimensions = " - ".join( @@ -464,11 +481,14 @@ def __init__(self, gene: Generate): def __call__(self, config_file: str, layer_name: str) -> TileStore | None: """Get the tilestore based on the layername config file any layer type.""" config = self.gene._gene.get_config(config_file) + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config_file) + return None layer = config.config["layers"][layer_name] if layer["type"] == "wms": params = layer.get("params", {}).copy() if "STYLES" not in params: - params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(",")) + params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer.get("layers", "").split(",")) if layer.get("generate_salt", False): params["SALT"] = str(random.randint(0, 999999)) # nosec # noqa: S311 @@ -478,7 +498,7 @@ def __call__(self, config_file: str, layer_name: str) -> TileStore | None: tilelayouts=( WMSTileLayout( url=layer["url"], - layers=layer["layers"], + layers=layer.get("layers", ""), srs=config.config["grids"][layer["grid"]].get("srs", configuration.SRS_DEFAULT), format_pattern=layer["mime_type"], border=( @@ -667,7 +687,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: else: if options.config: for layer in config.config["generation"].get( - "default_layers", config.config["layers"].keys() + "default_layers", config.config.get("layers", {}).keys() ): generate.gene(layer) except tilecloud.filter.error.TooManyErrors: @@ -697,7 +717,11 @@ def __init__(self, gene: Generate, meta: bool, count: Count): def __call__(self, config_file: str, layer_name: str) -> Callable[[Tile], Tile | None]: """Call.""" - layer = self.gene._gene.get_config(config_file).config["layers"][layer_name] + config = self.gene._gene.get_config(config_file) + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config_file) + return lambda tile: tile + layer = config.config["layers"][layer_name] conf_name = "empty_metatile_detection" if self.meta else "empty_tile_detection" if conf_name in layer: empty_tile = layer["empty_metatile_detection"] if self.meta else layer["empty_tile_detection"] diff --git a/tilecloud_chain/server.py b/tilecloud_chain/server.py index 89f1e71ef..34307023d 100644 --- a/tilecloud_chain/server.py +++ b/tilecloud_chain/server.py @@ -197,7 +197,7 @@ def get_cache(self, config: tilecloud_chain.DatedConfig) -> tilecloud_chain.conf @staticmethod def get_layers(config: tilecloud_chain.DatedConfig) -> list[str]: """Get the layer from the config.""" - layers: list[str] = cast(list[str], config.config["layers"].keys()) + layers: list[str] = cast(list[str], config.config.get("layers", {}).keys()) return config.config["server"].get("layers", layers) def get_filter( @@ -220,7 +220,7 @@ def get_filter( self.filter_cache.setdefault(config.file, {})[layer_name] = DatedFilter(layer_filter, config.mtime) return layer_filter - def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> tilecloud.TileStore: + def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> tilecloud.TileStore | None: """Get the store from the config.""" dated_store = self.store_cache.get(config.file, {}).get(layer_name) @@ -230,12 +230,17 @@ def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> til assert _TILEGENERATION store = _TILEGENERATION.get_store(config, self.get_cache(config), layer_name, read_only=True) + if store is None: + return None self.store_cache.setdefault(config.file, {})[layer_name] = DatedStore(store, config.mtime) return store @staticmethod def get_max_zoom_seed(config: tilecloud_chain.DatedConfig, layer_name: str) -> int: """Get the max zoom to be bet in the stored cache.""" + if layer_name not in config.config.get("layers", {}): + _LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config.file) + return 999999 layer = config.config["layers"][layer_name] if "min_resolution_seed" in layer: max_zoom_seed = -1 @@ -529,8 +534,8 @@ def serve( "SERVICE": "WMS", "VERSION": layer.get("version", "1.1.1"), "REQUEST": "GetFeatureInfo", - "LAYERS": layer["layers"], - "QUERY_LAYERS": layer["query_layers"], + "LAYERS": layer.get("layers", ""), + "QUERY_LAYERS": layer.get("query_layers", layer.get("layers", "")), "STYLES": params["STYLE"], "FORMAT": params["FORMAT"], "INFO_FORMAT": params["INFO_FORMAT"], diff --git a/tilecloud_chain/tests/__init__.py b/tilecloud_chain/tests/__init__.py index dd5769654..c0aa5e042 100644 --- a/tilecloud_chain/tests/__init__.py +++ b/tilecloud_chain/tests/__init__.py @@ -105,7 +105,7 @@ def assert_main_equals( for expect in expected: if os.path.exists(expect[0]): os.remove(expect[0]) - if type(cmd) == list: + if type(cmd) is list: sys.argv = cmd else: sys.argv = re.sub(" +", " ", cmd).split(" ")