Merge pull request #2678 from camptocamp/index-error

Be more careful on getting layers

tilecloud_chain/__init__.py

@@ -26,7 +26,7 @@
 from io import BytesIO
 from itertools import product
 from math import ceil, sqrt
-from typing import IO, TYPE_CHECKING, Any, TextIO, TypedDict, cast
+from typing import IO, TYPE_CHECKING, Any, Literal, TextIO, TypedDict, cast
 
 import boto3
 import botocore.client
@@ -833,8 +833,11 @@ def get_store(
         cache: tilecloud_chain.configuration.Cache,
         layer_name: str,
         read_only: bool = False,
-    ) -> TileStore:
+    ) -> TileStore | None:
         """Get the tile store."""
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
+            return None
         layer = config.config["layers"][layer_name]
         grid = config.config["grids"][layer["grid"]]
         layout = WMTSTileLayout(
@@ -928,7 +931,7 @@ def get_tilesstore(self, cache: str | None = None) -> TimedTileStoreWrapper:
         """Get the tile store."""
         gene = self
 
-        def get_store(config_file: str, layer_name: str) -> TileStore:
+        def get_store(config_file: str, layer_name: str) -> TileStore | None:
             config = gene.get_config(config_file)
             cache_name = cache or config.config["generation"].get(
                 "default_cache", configuration.DEFAULT_CACHE_DEFAULT
@@ -972,6 +975,9 @@ def add_metatile_splitter(self, store: TileStore | None = None) -> None:
 
             def get_splitter(config_file: str, layer_name: str) -> MetaTileSplitterTileStore | None:
                 config = gene.get_config(config_file)
+                if layer_name not in config.config.get("layers", {}):
+                    _LOGGER.warning("Layer %s not found in config %s", layer_name, config_file)
+                    return None
                 layer = config.config["layers"][layer_name]
                 if layer.get("meta"):
                     return MetaTileSplitterTileStore(
@@ -1098,6 +1104,9 @@ def get_geoms(
         if dated_geoms is not None and config.mtime == dated_geoms.mtime:
             return dated_geoms.geoms
 
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
+            return {}
         layer = config.config["layers"][layer_name]
 
         if self.options.near is not None or (
@@ -1187,6 +1196,9 @@ def get_geoms(
 
     def init_tilecoords(self, config: DatedConfig, layer_name: str) -> None:
         """Initialize the tilestream for the given layer."""
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
+            return
         layer = config.config["layers"][layer_name]
         resolutions = config.config["grids"][layer["grid"]]["resolutions"]
 
@@ -1296,6 +1308,9 @@ def _tilestream(
     def set_tilecoords(self, config: DatedConfig, tilecoords: Iterable[TileCoord], layer_name: str) -> None:
         """Set the tilestream for the given tilecoords."""
         assert tilecoords is not None
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
+            return
         layer = config.config["layers"][layer_name]
 
         metadata = {"layer": layer_name, "config_file": config.file}
@@ -1327,6 +1342,9 @@ def process(self, name: str | None = None, key: str = "post_process") -> None:
 
         def get_process(config_file: str, layer_name: str) -> Process | None:
             config = gene.get_config(config_file)
+            if layer_name not in config.config.get("layers", {}):
+                _LOGGER.warning("Layer %s not found in config %s", layer_name, config_file)
+                return None
             layer = config.config["layers"][layer_name]
             name_ = name
             if name_ is None:
@@ -1575,7 +1593,7 @@ def __init__(
         get_action: Callable[[str, str], Callable[[Tile], Tile | None] | None],
     ) -> None:
         self.get_action = get_action
-        self.actions: dict[tuple[str, str], _DatedAction | None] = {}
+        self.actions: dict[tuple[str, str], _DatedAction] = {}
 
     def _get_action(self, config_file: str, layer: str) -> Callable[[Tile], Tile | None] | None:
         """Get the action based on the tile's layer name."""
@@ -1606,13 +1624,13 @@ def __call__(self, tile: Tile) -> Tile | None:
 
     def __str__(self) -> str:
         """Return a string representation of the object."""
-        actions = {str(action) for action in self.actions.values()}
+        actions = {str(action.action) for action in self.actions.values()}
         keys = {f"{config_file}:{layer}" for config_file, layer in self.actions}
         return f"{self.__class__.__name__}({', '.join(actions)} - {', '.join(keys)})"
 
     def __repr__(self) -> str:
         """Return a string representation of the object."""
-        actions = {repr(action) for action in self.actions.values()}
+        actions = {repr(action.action) for action in self.actions.values()}
         keys = {f"{config_file}:{layer}" for config_file, layer in self.actions}
         return f"{self.__class__.__name__}({', '.join(actions)} - {', '.join(keys)})"
 
@@ -1689,6 +1707,9 @@ def filter_tilecoord(
         self, config: DatedConfig, tilecoord: TileCoord, layer_name: str, host: str | None = None
     ) -> bool:
         """Filter the tilecoord."""
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
+            return True
         layer = config.config["layers"][layer_name]
         grid_name = layer["grid"]
         grid = config.config["grids"][grid_name]
@@ -1780,7 +1801,15 @@ class Process:
 
     def __init__(self, config: tilecloud_chain.configuration.ProcessCommand, options: Namespace) -> None:
         self.config = config
-        self.options = options
+        self.options: list[Literal["verbose"] | Literal["debug"] | Literal["quiet"] | Literal["default"]] = []
+        if options.verbose:
+            self.options.append("verbose")
+        if options.debug:
+            self.options.append("debug")
+        if options.quiet:
+            self.options.append("quiet")
+        if not self.options:
+            self.options.append("default")
 
     def __call__(self, tile: Tile) -> Tile | None:
         """Process the tile."""
@@ -1791,16 +1820,9 @@ def __call__(self, tile: Tile) -> Tile | None:
 
             for cmd in self.config:
                 args = []
-                if (
-                    not self.options.verbose and not self.options.debug and not self.options.quiet
-                ) and "default" in cmd["arg"]:
-                    args.append(cmd["arg"]["default"])
-                if self.options.verbose and "verbose" in cmd["arg"]:
-                    args.append(cmd["arg"]["verbose"])
-                if self.options.debug and "debug" in cmd["arg"]:
-                    args.append(cmd["arg"]["debug"])
-                if self.options.quiet and "quiet" in cmd["arg"]:
-                    args.append(cmd["arg"]["quiet"])
+                for option in self.options:
+                    if option in cmd["arg"]:
+                        args.append(cmd["arg"][option])
 
                 if cmd.get("need_out", configuration.NEED_OUT_DEFAULT):
                     fd_out, name_out = tempfile.mkstemp()
@@ -1843,6 +1865,12 @@ def __call__(self, tile: Tile) -> Tile | None:
 
         return tile
 
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}({', '.join(self.options)} - {self.config})"
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
 
 class TilesFileStore(TileStore):
     """Load tiles to be generate from a file."""

tilecloud_chain/controller.py

@@ -216,7 +216,7 @@ def get_wmts_capabilities(
             str,
             jinja2_template(
                 data.decode("utf-8"),
-                layers=config.config["layers"],
+                layers=config.config.get("layers", {}),
                 layer_legends=gene.layer_legends,
                 grids=config.config["grids"],
                 getcapabilities=urljoin(  # type: ignore
@@ -304,7 +304,7 @@ def _fill_legend(
     with concurrent.futures.ThreadPoolExecutor(
         max_workers=int(os.environ.get("TILECLOUD_CHAIN_CONCURRENT_GET_LEGEND", "10"))
     ) as executor:
-        for layer_name, layer in config.config["layers"].items():
+        for layer_name, layer in config.config.get("layers", {}).items():
             if (
                 "legend_mime" in layer
                 and "legend_extension" in layer
@@ -341,7 +341,7 @@ def _fill_legend(
 
     _LOGGER.debug("Get %i legend images in %s", len(legend_image_future), time.perf_counter() - start)
 
-    for layer_name, layer in config.config["layers"].items():
+    for layer_name, layer in config.config.get("layers", {}).items():
         previous_legend: tilecloud_chain.Legend | None = None
         previous_resolution = None
         if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends:
@@ -472,14 +472,14 @@ def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) ->
     config = gene.get_config(gene.config_file)
     cache = config.config["caches"][gene.options.cache]
 
-    for layer_name, layer in config.config["layers"].items():
+    for layer_name, layer in config.config.get("layers", {}).items():
         if "legend_mime" in layer and "legend_extension" in layer and layer["type"] == "wms":
             session = requests.session()
             session.headers.update(layer["headers"])
             previous_hash = None
             for zoom, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]):
                 legends = []
-                for wms_layer in layer["layers"].split(","):
+                for wms_layer in layer.get("layers", "").split(","):
                     url = (
                         layer["url"]
                         + "?"

tilecloud_chain/generate.py

@@ -195,6 +195,11 @@ def _generate_queue(self, layer_name: str | None) -> None:
         assert layer_name is not None
         assert self._gene.config_file is not None
         config = self._gene.get_config(self._gene.config_file)
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning(
+                "Layer '%s' not found in the configuration file '%s'", layer_name, self._gene.config_file
+            )
+            sys.exit(1)
         layer = config.config["layers"][layer_name]
 
         if self._options.get_bbox:
@@ -215,6 +220,11 @@ def _generate_queue(self, layer_name: str | None) -> None:
             self._gene.init_tilecoords(config, layer_name)
 
         elif self._options.role == "hash":
+            if layer_name not in config.config.get("layers", {}):
+                _LOGGER.warning(
+                    "Layer '%s' not found in the configuration file '%s'", layer_name, self._gene.config_file
+                )
+                sys.exit(1)
             layer = config.config["layers"][layer_name]
             try:
                 z, x, y = (int(v) for v in self._options.get_hash.split("/"))
@@ -371,6 +381,13 @@ def generate_resume(self, layer_name: str | None) -> None:
             layer = None
             if layer_name is not None:
                 assert config is not None
+                if layer_name not in config.config.get("layers", {}):
+                    _LOGGER.warning(
+                        "Layer '%s' not found in the configuration file '%s'",
+                        layer_name,
+                        self._gene.config_file,
+                    )
+                    sys.exit(1)
                 layer = config.config["layers"][layer_name]
                 all_dimensions = self._gene.get_all_dimensions(layer)
                 formatted_dimensions = " - ".join(
@@ -464,11 +481,14 @@ def __init__(self, gene: Generate):
     def __call__(self, config_file: str, layer_name: str) -> TileStore | None:
         """Get the tilestore based on the layername config file any layer type."""
         config = self.gene._gene.get_config(config_file)
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config_file)
+            return None
         layer = config.config["layers"][layer_name]
         if layer["type"] == "wms":
             params = layer.get("params", {}).copy()
             if "STYLES" not in params:
-                params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(","))
+                params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer.get("layers", "").split(","))
             if layer.get("generate_salt", False):
                 params["SALT"] = str(random.randint(0, 999999))  # nosec # noqa: S311
 
@@ -478,7 +498,7 @@ def __call__(self, config_file: str, layer_name: str) -> TileStore | None:
                     tilelayouts=(
                         WMSTileLayout(
                             url=layer["url"],
-                            layers=layer["layers"],
+                            layers=layer.get("layers", ""),
                             srs=config.config["grids"][layer["grid"]].get("srs", configuration.SRS_DEFAULT),
                             format_pattern=layer["mime_type"],
                             border=(
@@ -667,7 +687,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None:
             else:
                 if options.config:
                     for layer in config.config["generation"].get(
-                        "default_layers", config.config["layers"].keys()
+                        "default_layers", config.config.get("layers", {}).keys()
                     ):
                         generate.gene(layer)
         except tilecloud.filter.error.TooManyErrors:
@@ -697,7 +717,11 @@ def __init__(self, gene: Generate, meta: bool, count: Count):
 
     def __call__(self, config_file: str, layer_name: str) -> Callable[[Tile], Tile | None]:
         """Call."""
-        layer = self.gene._gene.get_config(config_file).config["layers"][layer_name]
+        config = self.gene._gene.get_config(config_file)
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config_file)
+            return lambda tile: tile
+        layer = config.config["layers"][layer_name]
         conf_name = "empty_metatile_detection" if self.meta else "empty_tile_detection"
         if conf_name in layer:
             empty_tile = layer["empty_metatile_detection"] if self.meta else layer["empty_tile_detection"]

tilecloud_chain/server.py

@@ -197,7 +197,7 @@ def get_cache(self, config: tilecloud_chain.DatedConfig) -> tilecloud_chain.conf
     @staticmethod
     def get_layers(config: tilecloud_chain.DatedConfig) -> list[str]:
         """Get the layer from the config."""
-        layers: list[str] = cast(list[str], config.config["layers"].keys())
+        layers: list[str] = cast(list[str], config.config.get("layers", {}).keys())
         return config.config["server"].get("layers", layers)
 
     def get_filter(
@@ -220,7 +220,7 @@ def get_filter(
         self.filter_cache.setdefault(config.file, {})[layer_name] = DatedFilter(layer_filter, config.mtime)
         return layer_filter
 
-    def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> tilecloud.TileStore:
+    def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> tilecloud.TileStore | None:
         """Get the store from the config."""
         dated_store = self.store_cache.get(config.file, {}).get(layer_name)
 
@@ -230,12 +230,17 @@ def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> til
         assert _TILEGENERATION
 
         store = _TILEGENERATION.get_store(config, self.get_cache(config), layer_name, read_only=True)
+        if store is None:
+            return None
         self.store_cache.setdefault(config.file, {})[layer_name] = DatedStore(store, config.mtime)
         return store
 
     @staticmethod
     def get_max_zoom_seed(config: tilecloud_chain.DatedConfig, layer_name: str) -> int:
         """Get the max zoom to be bet in the stored cache."""
+        if layer_name not in config.config.get("layers", {}):
+            _LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config.file)
+            return 999999
         layer = config.config["layers"][layer_name]
         if "min_resolution_seed" in layer:
             max_zoom_seed = -1
@@ -529,8 +534,8 @@ def serve(
                                 "SERVICE": "WMS",
                                 "VERSION": layer.get("version", "1.1.1"),
                                 "REQUEST": "GetFeatureInfo",
-                                "LAYERS": layer["layers"],
-                                "QUERY_LAYERS": layer["query_layers"],
+                                "LAYERS": layer.get("layers", ""),
+                                "QUERY_LAYERS": layer.get("query_layers", layer.get("layers", "")),
                                 "STYLES": params["STYLE"],
                                 "FORMAT": params["FORMAT"],
                                 "INFO_FORMAT": params["INFO_FORMAT"],

tilecloud_chain/tests/__init__.py

@@ -105,7 +105,7 @@ def assert_main_equals(
             for expect in expected:
                 if os.path.exists(expect[0]):
                     os.remove(expect[0])
-        if type(cmd) == list:
+        if type(cmd) is list:
             sys.argv = cmd
         else:
             sys.argv = re.sub(" +", " ", cmd).split(" ")