Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Be more careful on getting layers #2678

Merged
merged 1 commit into from
Dec 20, 2024
Merged

Be more careful on getting layers #2678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 45 additions & 17 deletions tilecloud_chain/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from io import BytesIO
from itertools import product
from math import ceil, sqrt
from typing import IO, TYPE_CHECKING, Any, TextIO, TypedDict, cast
from typing import IO, TYPE_CHECKING, Any, Literal, TextIO, TypedDict, cast

import boto3
import botocore.client
Expand Down Expand Up @@ -833,8 +833,11 @@ def get_store(
cache: tilecloud_chain.configuration.Cache,
layer_name: str,
read_only: bool = False,
) -> TileStore:
) -> TileStore | None:
"""Get the tile store."""
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
return None
layer = config.config["layers"][layer_name]
grid = config.config["grids"][layer["grid"]]
layout = WMTSTileLayout(
Expand Down Expand Up @@ -928,7 +931,7 @@ def get_tilesstore(self, cache: str | None = None) -> TimedTileStoreWrapper:
"""Get the tile store."""
gene = self

def get_store(config_file: str, layer_name: str) -> TileStore:
def get_store(config_file: str, layer_name: str) -> TileStore | None:
config = gene.get_config(config_file)
cache_name = cache or config.config["generation"].get(
"default_cache", configuration.DEFAULT_CACHE_DEFAULT
Expand Down Expand Up @@ -972,6 +975,9 @@ def add_metatile_splitter(self, store: TileStore | None = None) -> None:

def get_splitter(config_file: str, layer_name: str) -> MetaTileSplitterTileStore | None:
config = gene.get_config(config_file)
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config_file)
return None
layer = config.config["layers"][layer_name]
if layer.get("meta"):
return MetaTileSplitterTileStore(
Expand Down Expand Up @@ -1098,6 +1104,9 @@ def get_geoms(
if dated_geoms is not None and config.mtime == dated_geoms.mtime:
return dated_geoms.geoms

if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
return {}
layer = config.config["layers"][layer_name]

if self.options.near is not None or (
Expand Down Expand Up @@ -1187,6 +1196,9 @@ def get_geoms(

def init_tilecoords(self, config: DatedConfig, layer_name: str) -> None:
"""Initialize the tilestream for the given layer."""
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
return
layer = config.config["layers"][layer_name]
resolutions = config.config["grids"][layer["grid"]]["resolutions"]

Expand Down Expand Up @@ -1296,6 +1308,9 @@ def _tilestream(
def set_tilecoords(self, config: DatedConfig, tilecoords: Iterable[TileCoord], layer_name: str) -> None:
"""Set the tilestream for the given tilecoords."""
assert tilecoords is not None
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
return
layer = config.config["layers"][layer_name]

metadata = {"layer": layer_name, "config_file": config.file}
Expand Down Expand Up @@ -1327,6 +1342,9 @@ def process(self, name: str | None = None, key: str = "post_process") -> None:

def get_process(config_file: str, layer_name: str) -> Process | None:
config = gene.get_config(config_file)
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config_file)
return None
layer = config.config["layers"][layer_name]
name_ = name
if name_ is None:
Expand Down Expand Up @@ -1575,7 +1593,7 @@ def __init__(
get_action: Callable[[str, str], Callable[[Tile], Tile | None] | None],
) -> None:
self.get_action = get_action
self.actions: dict[tuple[str, str], _DatedAction | None] = {}
self.actions: dict[tuple[str, str], _DatedAction] = {}

def _get_action(self, config_file: str, layer: str) -> Callable[[Tile], Tile | None] | None:
"""Get the action based on the tile's layer name."""
Expand Down Expand Up @@ -1606,13 +1624,13 @@ def __call__(self, tile: Tile) -> Tile | None:

def __str__(self) -> str:
"""Return a string representation of the object."""
actions = {str(action) for action in self.actions.values()}
actions = {str(action.action) for action in self.actions.values()}
keys = {f"{config_file}:{layer}" for config_file, layer in self.actions}
return f"{self.__class__.__name__}({', '.join(actions)} - {', '.join(keys)})"

def __repr__(self) -> str:
"""Return a string representation of the object."""
actions = {repr(action) for action in self.actions.values()}
actions = {repr(action.action) for action in self.actions.values()}
keys = {f"{config_file}:{layer}" for config_file, layer in self.actions}
return f"{self.__class__.__name__}({', '.join(actions)} - {', '.join(keys)})"

Expand Down Expand Up @@ -1689,6 +1707,9 @@ def filter_tilecoord(
self, config: DatedConfig, tilecoord: TileCoord, layer_name: str, host: str | None = None
) -> bool:
"""Filter the tilecoord."""
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer %s not found in config %s", layer_name, config.file)
return True
layer = config.config["layers"][layer_name]
grid_name = layer["grid"]
grid = config.config["grids"][grid_name]
Expand Down Expand Up @@ -1780,7 +1801,15 @@ class Process:

def __init__(self, config: tilecloud_chain.configuration.ProcessCommand, options: Namespace) -> None:
self.config = config
self.options = options
self.options: list[Literal["verbose"] | Literal["debug"] | Literal["quiet"] | Literal["default"]] = []
if options.verbose:
self.options.append("verbose")
if options.debug:
self.options.append("debug")
if options.quiet:
self.options.append("quiet")
if not self.options:
self.options.append("default")

def __call__(self, tile: Tile) -> Tile | None:
"""Process the tile."""
Expand All @@ -1791,16 +1820,9 @@ def __call__(self, tile: Tile) -> Tile | None:

for cmd in self.config:
args = []
if (
not self.options.verbose and not self.options.debug and not self.options.quiet
) and "default" in cmd["arg"]:
args.append(cmd["arg"]["default"])
if self.options.verbose and "verbose" in cmd["arg"]:
args.append(cmd["arg"]["verbose"])
if self.options.debug and "debug" in cmd["arg"]:
args.append(cmd["arg"]["debug"])
if self.options.quiet and "quiet" in cmd["arg"]:
args.append(cmd["arg"]["quiet"])
for option in self.options:
if option in cmd["arg"]:
args.append(cmd["arg"][option])

if cmd.get("need_out", configuration.NEED_OUT_DEFAULT):
fd_out, name_out = tempfile.mkstemp()
Expand Down Expand Up @@ -1843,6 +1865,12 @@ def __call__(self, tile: Tile) -> Tile | None:

return tile

def __str__(self) -> str:
return f"{self.__class__.__name__}({', '.join(self.options)} - {self.config})"

def __repr__(self) -> str:
return self.__str__()


class TilesFileStore(TileStore):
"""Load tiles to be generate from a file."""
Expand Down
10 changes: 5 additions & 5 deletions tilecloud_chain/controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ def get_wmts_capabilities(
str,
jinja2_template(
data.decode("utf-8"),
layers=config.config["layers"],
layers=config.config.get("layers", {}),
layer_legends=gene.layer_legends,
grids=config.config["grids"],
getcapabilities=urljoin( # type: ignore
Expand Down Expand Up @@ -304,7 +304,7 @@ def _fill_legend(
with concurrent.futures.ThreadPoolExecutor(
max_workers=int(os.environ.get("TILECLOUD_CHAIN_CONCURRENT_GET_LEGEND", "10"))
) as executor:
for layer_name, layer in config.config["layers"].items():
for layer_name, layer in config.config.get("layers", {}).items():
if (
"legend_mime" in layer
and "legend_extension" in layer
Expand Down Expand Up @@ -341,7 +341,7 @@ def _fill_legend(

_LOGGER.debug("Get %i legend images in %s", len(legend_image_future), time.perf_counter() - start)

for layer_name, layer in config.config["layers"].items():
for layer_name, layer in config.config.get("layers", {}).items():
previous_legend: tilecloud_chain.Legend | None = None
previous_resolution = None
if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends:
Expand Down Expand Up @@ -472,14 +472,14 @@ def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) ->
config = gene.get_config(gene.config_file)
cache = config.config["caches"][gene.options.cache]

for layer_name, layer in config.config["layers"].items():
for layer_name, layer in config.config.get("layers", {}).items():
if "legend_mime" in layer and "legend_extension" in layer and layer["type"] == "wms":
session = requests.session()
session.headers.update(layer["headers"])
previous_hash = None
for zoom, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]):
legends = []
for wms_layer in layer["layers"].split(","):
for wms_layer in layer.get("layers", "").split(","):
url = (
layer["url"]
+ "?"
Expand Down
32 changes: 28 additions & 4 deletions tilecloud_chain/generate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,11 @@ def _generate_queue(self, layer_name: str | None) -> None:
assert layer_name is not None
assert self._gene.config_file is not None
config = self._gene.get_config(self._gene.config_file)
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning(
"Layer '%s' not found in the configuration file '%s'", layer_name, self._gene.config_file
)
sys.exit(1)
layer = config.config["layers"][layer_name]

if self._options.get_bbox:
Expand All @@ -215,6 +220,11 @@ def _generate_queue(self, layer_name: str | None) -> None:
self._gene.init_tilecoords(config, layer_name)

elif self._options.role == "hash":
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning(
"Layer '%s' not found in the configuration file '%s'", layer_name, self._gene.config_file
)
sys.exit(1)
layer = config.config["layers"][layer_name]
try:
z, x, y = (int(v) for v in self._options.get_hash.split("/"))
Expand Down Expand Up @@ -371,6 +381,13 @@ def generate_resume(self, layer_name: str | None) -> None:
layer = None
if layer_name is not None:
assert config is not None
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning(
"Layer '%s' not found in the configuration file '%s'",
layer_name,
self._gene.config_file,
)
sys.exit(1)
layer = config.config["layers"][layer_name]
all_dimensions = self._gene.get_all_dimensions(layer)
formatted_dimensions = " - ".join(
Expand Down Expand Up @@ -464,11 +481,14 @@ def __init__(self, gene: Generate):
def __call__(self, config_file: str, layer_name: str) -> TileStore | None:
"""Get the tilestore based on the layername config file any layer type."""
config = self.gene._gene.get_config(config_file)
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config_file)
return None
layer = config.config["layers"][layer_name]
if layer["type"] == "wms":
params = layer.get("params", {}).copy()
if "STYLES" not in params:
params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(","))
params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer.get("layers", "").split(","))
if layer.get("generate_salt", False):
params["SALT"] = str(random.randint(0, 999999)) # nosec # noqa: S311

Expand All @@ -478,7 +498,7 @@ def __call__(self, config_file: str, layer_name: str) -> TileStore | None:
tilelayouts=(
WMSTileLayout(
url=layer["url"],
layers=layer["layers"],
layers=layer.get("layers", ""),
srs=config.config["grids"][layer["grid"]].get("srs", configuration.SRS_DEFAULT),
format_pattern=layer["mime_type"],
border=(
Expand Down Expand Up @@ -667,7 +687,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None:
else:
if options.config:
for layer in config.config["generation"].get(
"default_layers", config.config["layers"].keys()
"default_layers", config.config.get("layers", {}).keys()
):
generate.gene(layer)
except tilecloud.filter.error.TooManyErrors:
Expand Down Expand Up @@ -697,7 +717,11 @@ def __init__(self, gene: Generate, meta: bool, count: Count):

def __call__(self, config_file: str, layer_name: str) -> Callable[[Tile], Tile | None]:
"""Call."""
layer = self.gene._gene.get_config(config_file).config["layers"][layer_name]
config = self.gene._gene.get_config(config_file)
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config_file)
return lambda tile: tile
layer = config.config["layers"][layer_name]
conf_name = "empty_metatile_detection" if self.meta else "empty_tile_detection"
if conf_name in layer:
empty_tile = layer["empty_metatile_detection"] if self.meta else layer["empty_tile_detection"]
Expand Down
13 changes: 9 additions & 4 deletions tilecloud_chain/server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ def get_cache(self, config: tilecloud_chain.DatedConfig) -> tilecloud_chain.conf
@staticmethod
def get_layers(config: tilecloud_chain.DatedConfig) -> list[str]:
"""Get the layer from the config."""
layers: list[str] = cast(list[str], config.config["layers"].keys())
layers: list[str] = cast(list[str], config.config.get("layers", {}).keys())
return config.config["server"].get("layers", layers)

def get_filter(
Expand All @@ -220,7 +220,7 @@ def get_filter(
self.filter_cache.setdefault(config.file, {})[layer_name] = DatedFilter(layer_filter, config.mtime)
return layer_filter

def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> tilecloud.TileStore:
def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> tilecloud.TileStore | None:
"""Get the store from the config."""
dated_store = self.store_cache.get(config.file, {}).get(layer_name)

Expand All @@ -230,12 +230,17 @@ def get_store(self, config: tilecloud_chain.DatedConfig, layer_name: str) -> til
assert _TILEGENERATION

store = _TILEGENERATION.get_store(config, self.get_cache(config), layer_name, read_only=True)
if store is None:
return None
self.store_cache.setdefault(config.file, {})[layer_name] = DatedStore(store, config.mtime)
return store

@staticmethod
def get_max_zoom_seed(config: tilecloud_chain.DatedConfig, layer_name: str) -> int:
"""Get the max zoom to be bet in the stored cache."""
if layer_name not in config.config.get("layers", {}):
_LOGGER.warning("Layer '%s' not found in the configuration file '%s'", layer_name, config.file)
return 999999
layer = config.config["layers"][layer_name]
if "min_resolution_seed" in layer:
max_zoom_seed = -1
Expand Down Expand Up @@ -529,8 +534,8 @@ def serve(
"SERVICE": "WMS",
"VERSION": layer.get("version", "1.1.1"),
"REQUEST": "GetFeatureInfo",
"LAYERS": layer["layers"],
"QUERY_LAYERS": layer["query_layers"],
"LAYERS": layer.get("layers", ""),
"QUERY_LAYERS": layer.get("query_layers", layer.get("layers", "")),
"STYLES": params["STYLE"],
"FORMAT": params["FORMAT"],
"INFO_FORMAT": params["INFO_FORMAT"],
Expand Down
2 changes: 1 addition & 1 deletion tilecloud_chain/tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def assert_main_equals(
for expect in expected:
if os.path.exists(expect[0]):
os.remove(expect[0])
if type(cmd) == list:
if type(cmd) is list:
sys.argv = cmd
else:
sys.argv = re.sub(" +", " ", cmd).split(" ")
Expand Down
Loading