From a71a0e0c9f0484f3b05ea643374b7ae821fa208d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 30 Nov 2024 20:23:39 +0100 Subject: [PATCH 1/2] Add missing output --- tilecloud_chain/CONFIG.md | 2 +- tilecloud_chain/__init__.py | 16 +++-- tilecloud_chain/configuration.py | 2 - tilecloud_chain/controller.py | 98 +++++++++++++++-------------- tilecloud_chain/generate.py | 1 + tilecloud_chain/schema.json | 1 - tilecloud_chain/store/postgresql.py | 6 +- 7 files changed, 69 insertions(+), 57 deletions(-) diff --git a/tilecloud_chain/CONFIG.md b/tilecloud_chain/CONFIG.md index 3bdb2bb08..45fd88a9d 100644 --- a/tilecloud_chain/CONFIG.md +++ b/tilecloud_chain/CONFIG.md @@ -298,7 +298,7 @@ - **`name`** _(string)_: The name used in the admin interface. - **`allowed_commands`** _(array)_: The allowed commands (main configuration). Default: `["generate-tiles", "generate-controller", "generate-cost"]`. - **Items** _(string)_ - - **`allowed_arguments`** _(array)_: The allowed arguments (main configuration). Default: `["--layer", "--get-hash", "--generate-legend-images", "--get-bbox", "--help", "--ignore-error", "--bbox", "--zoom", "--test", "--near", "--time", "--measure-generation-time", "--no-geom", "--dimensions"]`. + - **`allowed_arguments`** _(array)_: The allowed arguments (main configuration). Default: `["--layer", "--get-hash", "--generate-legend-images", "--get-bbox", "--ignore-error", "--bbox", "--zoom", "--test", "--near", "--time", "--measure-generation-time", "--no-geom", "--dimensions"]`. - **Items** _(string)_ - **`admin_footer`** _(string)_: The footer of the admin interface. - **`admin_footer_classes`** _(string)_: The CSS classes used on the footer of the admin interface. diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index f1f1c7c44..728bda535 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -209,9 +209,11 @@ def __init__( self, gene: "TileGeneration", functions: list[Callable[[Tile], Tile]], + out: IO[str] | None = None, ) -> None: self.gene = gene self.functions = functions + self.out = out self.safe = gene.options is None or not gene.options.debug daemon = gene.options is not None and getattr(gene.options, "daemon", False) self.max_consecutive_errors = ( @@ -268,7 +270,11 @@ def __call__(self, tile: Tile | None) -> Tile | None: ): assert isinstance(tile.error, str) tile.error = f"WMS server error: {self._re_rm_xml_tag.sub('', tile.error)}" - print(f"Error with tile {tile.tilecoord} {tile.formated_metadata}:\n{tile.error}") + if self.out: + print( + f"Error with tile {tile.tilecoord} {tile.formated_metadata}:\n{tile.error}", + file=self.out, + ) _LOGGER.warning( "Error with tile %s %s:\n%s", tile.tilecoord, tile.formated_metadata, tile.error ) @@ -435,6 +441,7 @@ def __init__( configure_logging: bool = True, multi_thread: bool = True, maxconsecutive_errors: bool = True, + out: IO[str] | None = None, ): self.geoms_cache: dict[str, dict[str, DatedGeoms]] = {} self._close_actions: list[Close] = [] @@ -446,6 +453,7 @@ def __init__( self.metatilesplitter_thread_pool: ThreadPoolExecutor | None = None self.multi_thread = multi_thread self.maxconsecutive_errors = maxconsecutive_errors + self.out = out self.grid_cache: dict[str, dict[str, DatedTileGrid]] = {} self.layer_legends: dict[str, list[Legend]] = {} self.config_file = config_file @@ -975,7 +983,7 @@ def get_splitter(config_file: str, layer_name: str) -> MetaTileSplitterTileStore store = TimedTileStoreWrapper(MultiTileStore(get_splitter), store_name="splitter") - run = Run(self, self.functions_tiles) + run = Run(self, self.functions_tiles, out=self.out) nb_thread = int(os.environ.get("TILE_NB_THREAD", "1")) if nb_thread == 1 or not self.multi_thread: @@ -1379,7 +1387,7 @@ def consume(self, test: int | None = None) -> None: start = datetime.now() - run = Run(self, self.functions_metatiles) + run = Run(self, self.functions_metatiles, out=self.out) if test is None: if TYPE_CHECKING: @@ -1579,7 +1587,7 @@ def __call__(self, tile: Tile) -> Tile: if ref is None: ref = px elif px != ref: - print("Error: image is not uniform.") + print("Error: image is not uniform.", file=self.out) _LOGGER.debug("Error: image is not uniform.") sys.exit(1) diff --git a/tilecloud_chain/configuration.py b/tilecloud_chain/configuration.py index d11fab9d6..2cc867b0e 100644 --- a/tilecloud_chain/configuration.py +++ b/tilecloud_chain/configuration.py @@ -13,7 +13,6 @@ "--get-hash", "--generate-legend-images", "--get-bbox", - "--help", "--ignore-error", "--bbox", "--zoom", @@ -2526,7 +2525,6 @@ class Server(TypedDict, total=False): - --get-hash - --generate-legend-images - --get-bbox - - --help - --ignore-error - --bbox - --zoom diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py index bd5c51d18..8c22f86cf 100644 --- a/tilecloud_chain/controller.py +++ b/tilecloud_chain/controller.py @@ -44,8 +44,6 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: """Generate the contextual file like the legends.""" - del out - try: parser = ArgumentParser( description="Used to generate the contextual file like the capabilities, the legends, " @@ -72,7 +70,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: ) options = parser.parse_args(args[1:] if args else sys.argv[1:]) - gene = TileGeneration(options.config, options, layer_name=options.layer) + gene = TileGeneration(options.config, options, layer_name=options.layer, out=out) assert gene.config_file config = gene.get_config(gene.config_file) @@ -94,7 +92,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: sys.exit(0) if options.legends: - _generate_legend_images(gene) + _generate_legend_images(gene, out) except SystemExit: raise @@ -362,7 +360,7 @@ def _fill_legend( previous_resolution = resolution -def _generate_legend_images(gene: TileGeneration) -> None: +def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) -> None: assert gene.config_file config = gene.get_config(gene.config_file) cache = config.config["caches"][gene.options.cache] @@ -374,7 +372,7 @@ def _generate_legend_images(gene: TileGeneration) -> None: previous_hash = None for zoom, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]): legends = [] - for wmslayer in layer["layers"].split(","): + for wms_layer in layer["layers"].split(","): url = ( layer["url"] + "?" @@ -383,7 +381,7 @@ def _generate_legend_images(gene: TileGeneration) -> None: "SERVICE": "WMS", "VERSION": layer.get("version", "1.0.0"), "REQUEST": "GetLegendGraphic", - "LAYER": wmslayer, + "LAYER": wms_layer, "FORMAT": layer["legend_mime"], "TRANSPARENT": "TRUE" if layer["legend_mime"] == "image/png" else "FALSE", "STYLE": layer["wmts_style"], @@ -394,64 +392,70 @@ def _generate_legend_images(gene: TileGeneration) -> None: _LOGGER.debug( "Get legend image for layer '%s'-'%s', resolution '%s': %s", layer_name, - wmslayer, + wms_layer, resolution, url, ) try: response = session.get(url) except Exception as e: # pylint: disable=broad-exception-caught - print( - "\n".join( - [ - f"Unable to get legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'", - url, - str(e), - ] + if out is not None: + print( + "\n".join( + [ + f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + str(e), + ] + ), + file=out, ) - ) _LOGGER.debug( "Unable to get legend image for layer '%s'-'%s', resolution '%s'", layer_name, - wmslayer, + wms_layer, resolution, exc_info=True, ) continue if response.status_code != 200: - print( - "\n".join( - [ - f"Unable to get legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'", - url, - f"status code: {response.status_code}: {response.reason}", - response.text, - ] + if out is not None: + print( + "\n".join( + [ + f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + f"status code: {response.status_code}: {response.reason}", + response.text, + ] + ), + file=out, ) - ) _LOGGER.debug( "Unable to get legend image for layer '%s'-'%s', resolution '%s': %s", layer_name, - wmslayer, + wms_layer, resolution, response.text, ) continue if not response.headers["Content-Type"].startswith(layer["legend_mime"].split("/")[0]): - print( - "\n".join( - [ - f"Unable to get legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'", - url, - f"Content-Type: {response.headers['Content-Type']}", - response.text, - ] + if out is not None: + print( + "\n".join( + [ + f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + f"Content-Type: {response.headers['Content-Type']}", + response.text, + ] + ), + file=out, ) - ) _LOGGER.debug( "Unable to get legend image for layer '%s'-'%s', resolution '%s', content-type: %s: %s", layer_name, - wmslayer, + wms_layer, resolution, response.headers["Content-Type"], response.text, @@ -460,19 +464,21 @@ def _generate_legend_images(gene: TileGeneration) -> None: try: legends.append(Image.open(BytesIO(response.content))) except Exception: # pylint: disable=broad-exception-caught - print( - "\n".join( - [ - f"Unable to read legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'", - url, - response.text, - ] + if out is not None: + print( + "\n".join( + [ + f"Unable to read legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + response.text, + ] + ), + file=out, ) - ) _LOGGER.debug( "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s", layer_name, - wmslayer, + wms_layer, resolution, response.text, exc_info=True, diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py index 0605d914c..f87656c61 100644 --- a/tilecloud_chain/generate.py +++ b/tilecloud_chain/generate.py @@ -611,6 +611,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: config_file=options.config or os.environ.get("TILEGENERATION_CONFIGFILE"), options=options, multi_thread=options.get_hash is None, + out=out, ) if ( diff --git a/tilecloud_chain/schema.json b/tilecloud_chain/schema.json index 0e061e5ff..5154ef6ad 100644 --- a/tilecloud_chain/schema.json +++ b/tilecloud_chain/schema.json @@ -1136,7 +1136,6 @@ "--get-hash", "--generate-legend-images", "--get-bbox", - "--help", "--ignore-error", "--bbox", "--zoom", diff --git a/tilecloud_chain/store/postgresql.py b/tilecloud_chain/store/postgresql.py index ca325ae9f..a6b85fbe9 100644 --- a/tilecloud_chain/store/postgresql.py +++ b/tilecloud_chain/store/postgresql.py @@ -233,13 +233,13 @@ def _start_job( _LOGGER.info("Running the command `%s` using the function directly", display_command) main(final_command, out) _LOGGER.info("Successfully ran the command `%s` using the function directly", display_command) - except Exception: # pylint: disable=broad-exception-caught - _LOGGER.exception("Error while running the command `%s`", display_command) - error = True except SystemExit as exception: if exception.code != 0: _LOGGER.exception("Error while running the command `%s`", display_command) error = True + except Exception: # pylint: disable=broad-exception-caught + _LOGGER.exception("Error while running the command `%s`", display_command) + error = True with SessionMaker() as session: job = session.query(Job).where(Job.id == job_id).with_for_update(of=Job).one() From 2a098bba3f1b742df55c57aedd3962fea0a4e68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sun, 1 Dec 2024 21:32:04 +0100 Subject: [PATCH 2/2] Refactor to reduce cognitive complexity --- tilecloud_chain/controller.py | 203 +++++++++++++++++++--------------- 1 file changed, 114 insertions(+), 89 deletions(-) diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py index 8c22f86cf..d354658e5 100644 --- a/tilecloud_chain/controller.py +++ b/tilecloud_chain/controller.py @@ -16,6 +16,7 @@ from urllib.parse import urlencode, urljoin import botocore.exceptions +import PIL.ImageFile import requests import ruamel.yaml import tilecloud.store.redis @@ -360,6 +361,112 @@ def _fill_legend( previous_resolution = resolution +def _get_legend_image( + layer_name: str, + wms_layer: str, + resolution: float, + url: str, + session: requests.Session, + main_mime_type: str, + out: IO[str] | None = None, +) -> PIL.ImageFile.ImageFile | None: + _LOGGER.debug( + "Get legend image for layer '%s'-'%s', resolution '%s': %s", + layer_name, + wms_layer, + resolution, + url, + ) + try: + response = session.get(url) + except Exception as e: # pylint: disable=broad-exception-caught + if out is not None: + print( + "\n".join( + [ + f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + str(e), + ] + ), + file=out, + ) + _LOGGER.debug( + "Unable to get legend image for layer '%s'-'%s', resolution '%s'", + layer_name, + wms_layer, + resolution, + exc_info=True, + ) + return None + if response.status_code != 200: + if out is not None: + print( + "\n".join( + [ + f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + f"status code: {response.status_code}: {response.reason}", + response.text, + ] + ), + file=out, + ) + _LOGGER.debug( + "Unable to get legend image for layer '%s'-'%s', resolution '%s': %s", + layer_name, + wms_layer, + resolution, + response.text, + ) + return None + if not response.headers["Content-Type"].startswith(main_mime_type): + if out is not None: + print( + "\n".join( + [ + f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + f"Content-Type: {response.headers['Content-Type']}", + response.text, + ] + ), + file=out, + ) + _LOGGER.debug( + "Unable to get legend image for layer '%s'-'%s', resolution '%s', content-type: %s: %s", + layer_name, + wms_layer, + resolution, + response.headers["Content-Type"], + response.text, + ) + return None + try: + return Image.open(BytesIO(response.content)) + except Exception: # pylint: disable=broad-exception-caught + if out is not None: + print( + "\n".join( + [ + f"Unable to read legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", + url, + response.text, + ] + ), + file=out, + ) + _LOGGER.debug( + "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s", + layer_name, + wms_layer, + resolution, + response.text, + exc_info=True, + ) + return None + + def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) -> None: assert gene.config_file config = gene.get_config(gene.config_file) @@ -389,100 +496,18 @@ def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) -> } ) ) - _LOGGER.debug( - "Get legend image for layer '%s'-'%s', resolution '%s': %s", + legend_image = _get_legend_image( layer_name, wms_layer, resolution, url, + session, + layer["legend_mime"].split("/")[0], + out, ) - try: - response = session.get(url) - except Exception as e: # pylint: disable=broad-exception-caught - if out is not None: - print( - "\n".join( - [ - f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", - url, - str(e), - ] - ), - file=out, - ) - _LOGGER.debug( - "Unable to get legend image for layer '%s'-'%s', resolution '%s'", - layer_name, - wms_layer, - resolution, - exc_info=True, - ) - continue - if response.status_code != 200: - if out is not None: - print( - "\n".join( - [ - f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", - url, - f"status code: {response.status_code}: {response.reason}", - response.text, - ] - ), - file=out, - ) - _LOGGER.debug( - "Unable to get legend image for layer '%s'-'%s', resolution '%s': %s", - layer_name, - wms_layer, - resolution, - response.text, - ) - continue - if not response.headers["Content-Type"].startswith(layer["legend_mime"].split("/")[0]): - if out is not None: - print( - "\n".join( - [ - f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", - url, - f"Content-Type: {response.headers['Content-Type']}", - response.text, - ] - ), - file=out, - ) - _LOGGER.debug( - "Unable to get legend image for layer '%s'-'%s', resolution '%s', content-type: %s: %s", - layer_name, - wms_layer, - resolution, - response.headers["Content-Type"], - response.text, - ) - continue - try: - legends.append(Image.open(BytesIO(response.content))) - except Exception: # pylint: disable=broad-exception-caught - if out is not None: - print( - "\n".join( - [ - f"Unable to read legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'", - url, - response.text, - ] - ), - file=out, - ) - _LOGGER.debug( - "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s", - layer_name, - wms_layer, - resolution, - response.text, - exc_info=True, - ) + if legend_image is not None: + legends.append(legend_image) + width = max(1, max(i.size[0] for i in legends) if legends else 0) height = max(1, sum(i.size[1] for i in legends) if legends else 0) image = Image.new("RGBA", (width, height))