Merge pull request #2623 from camptocamp/test-generated-legend-images

Add missing output
camptocamp · Dec 1, 2024 · c32a1e3 · c32a1e3
2 parents 418ceda + 2a098bb
commit c32a1e3
Show file tree

Hide file tree

Showing 7 changed files with 137 additions and 100 deletions.
diff --git a/tilecloud_chain/CONFIG.md b/tilecloud_chain/CONFIG.md
@@ -298,7 +298,7 @@
       - **`name`** _(string)_: The name used in the admin interface.
   - **`allowed_commands`** _(array)_: The allowed commands (main configuration). Default: `["generate-tiles", "generate-controller", "generate-cost"]`.
     - **Items** _(string)_
-  - **`allowed_arguments`** _(array)_: The allowed arguments (main configuration). Default: `["--layer", "--get-hash", "--generate-legend-images", "--get-bbox", "--help", "--ignore-error", "--bbox", "--zoom", "--test", "--near", "--time", "--measure-generation-time", "--no-geom", "--dimensions"]`.
+  - **`allowed_arguments`** _(array)_: The allowed arguments (main configuration). Default: `["--layer", "--get-hash", "--generate-legend-images", "--get-bbox", "--ignore-error", "--bbox", "--zoom", "--test", "--near", "--time", "--measure-generation-time", "--no-geom", "--dimensions"]`.
     - **Items** _(string)_
   - **`admin_footer`** _(string)_: The footer of the admin interface.
   - **`admin_footer_classes`** _(string)_: The CSS classes used on the footer of the admin interface.

diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py
@@ -209,9 +209,11 @@ def __init__(
         self,
         gene: "TileGeneration",
         functions: list[Callable[[Tile], Tile]],
+        out: IO[str] | None = None,
     ) -> None:
         self.gene = gene
         self.functions = functions
+        self.out = out
         self.safe = gene.options is None or not gene.options.debug
         daemon = gene.options is not None and getattr(gene.options, "daemon", False)
         self.max_consecutive_errors = (
@@ -268,7 +270,11 @@ def __call__(self, tile: Tile | None) -> Tile | None:
                     ):
                         assert isinstance(tile.error, str)
                         tile.error = f"WMS server error: {self._re_rm_xml_tag.sub('', tile.error)}"
-                    print(f"Error with tile {tile.tilecoord} {tile.formated_metadata}:\n{tile.error}")
+                    if self.out:
+                        print(
+                            f"Error with tile {tile.tilecoord} {tile.formated_metadata}:\n{tile.error}",
+                            file=self.out,
+                        )
                     _LOGGER.warning(
                         "Error with tile %s %s:\n%s", tile.tilecoord, tile.formated_metadata, tile.error
                     )
@@ -435,6 +441,7 @@ def __init__(
         configure_logging: bool = True,
         multi_thread: bool = True,
         maxconsecutive_errors: bool = True,
+        out: IO[str] | None = None,
     ):
         self.geoms_cache: dict[str, dict[str, DatedGeoms]] = {}
         self._close_actions: list[Close] = []
@@ -446,6 +453,7 @@ def __init__(
         self.metatilesplitter_thread_pool: ThreadPoolExecutor | None = None
         self.multi_thread = multi_thread
         self.maxconsecutive_errors = maxconsecutive_errors
+        self.out = out
         self.grid_cache: dict[str, dict[str, DatedTileGrid]] = {}
         self.layer_legends: dict[str, list[Legend]] = {}
         self.config_file = config_file
@@ -975,7 +983,7 @@ def get_splitter(config_file: str, layer_name: str) -> MetaTileSplitterTileStore
 
             store = TimedTileStoreWrapper(MultiTileStore(get_splitter), store_name="splitter")
 
-        run = Run(self, self.functions_tiles)
+        run = Run(self, self.functions_tiles, out=self.out)
         nb_thread = int(os.environ.get("TILE_NB_THREAD", "1"))
         if nb_thread == 1 or not self.multi_thread:
 
@@ -1379,7 +1387,7 @@ def consume(self, test: int | None = None) -> None:
 
         start = datetime.now()
 
-        run = Run(self, self.functions_metatiles)
+        run = Run(self, self.functions_metatiles, out=self.out)
 
         if test is None:
             if TYPE_CHECKING:
@@ -1579,7 +1587,7 @@ def __call__(self, tile: Tile) -> Tile:
             if ref is None:
                 ref = px
             elif px != ref:
-                print("Error: image is not uniform.")
+                print("Error: image is not uniform.", file=self.out)
                 _LOGGER.debug("Error: image is not uniform.")
                 sys.exit(1)
 

diff --git a/tilecloud_chain/configuration.py b/tilecloud_chain/configuration.py
@@ -13,7 +13,6 @@
     "--get-hash",
     "--generate-legend-images",
     "--get-bbox",
-    "--help",
     "--ignore-error",
     "--bbox",
     "--zoom",
@@ -2526,7 +2525,6 @@ class Server(TypedDict, total=False):
       - --get-hash
       - --generate-legend-images
       - --get-bbox
-      - --help
       - --ignore-error
       - --bbox
       - --zoom

diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py
@@ -16,6 +16,7 @@
 from urllib.parse import urlencode, urljoin
 
 import botocore.exceptions
+import PIL.ImageFile
 import requests
 import ruamel.yaml
 import tilecloud.store.redis
@@ -44,8 +45,6 @@
 
 def main(args: list[str] | None = None, out: IO[str] | None = None) -> None:
     """Generate the contextual file like the legends."""
-    del out
-
     try:
         parser = ArgumentParser(
             description="Used to generate the contextual file like the capabilities, the legends, "
@@ -72,7 +71,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None:
         )
 
         options = parser.parse_args(args[1:] if args else sys.argv[1:])
-        gene = TileGeneration(options.config, options, layer_name=options.layer)
+        gene = TileGeneration(options.config, options, layer_name=options.layer, out=out)
         assert gene.config_file
         config = gene.get_config(gene.config_file)
 
@@ -94,7 +93,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None:
             sys.exit(0)
 
         if options.legends:
-            _generate_legend_images(gene)
+            _generate_legend_images(gene, out)
 
     except SystemExit:
         raise
@@ -362,7 +361,113 @@ def _fill_legend(
                 previous_resolution = resolution
 
 
-def _generate_legend_images(gene: TileGeneration) -> None:
+def _get_legend_image(
+    layer_name: str,
+    wms_layer: str,
+    resolution: float,
+    url: str,
+    session: requests.Session,
+    main_mime_type: str,
+    out: IO[str] | None = None,
+) -> PIL.ImageFile.ImageFile | None:
+    _LOGGER.debug(
+        "Get legend image for layer '%s'-'%s', resolution '%s': %s",
+        layer_name,
+        wms_layer,
+        resolution,
+        url,
+    )
+    try:
+        response = session.get(url)
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        if out is not None:
+            print(
+                "\n".join(
+                    [
+                        f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'",
+                        url,
+                        str(e),
+                    ]
+                ),
+                file=out,
+            )
+        _LOGGER.debug(
+            "Unable to get legend image for layer '%s'-'%s', resolution '%s'",
+            layer_name,
+            wms_layer,
+            resolution,
+            exc_info=True,
+        )
+        return None
+    if response.status_code != 200:
+        if out is not None:
+            print(
+                "\n".join(
+                    [
+                        f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'",
+                        url,
+                        f"status code: {response.status_code}: {response.reason}",
+                        response.text,
+                    ]
+                ),
+                file=out,
+            )
+        _LOGGER.debug(
+            "Unable to get legend image for layer '%s'-'%s', resolution '%s': %s",
+            layer_name,
+            wms_layer,
+            resolution,
+            response.text,
+        )
+        return None
+    if not response.headers["Content-Type"].startswith(main_mime_type):
+        if out is not None:
+            print(
+                "\n".join(
+                    [
+                        f"Unable to get legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'",
+                        url,
+                        f"Content-Type: {response.headers['Content-Type']}",
+                        response.text,
+                    ]
+                ),
+                file=out,
+            )
+        _LOGGER.debug(
+            "Unable to get legend image for layer '%s'-'%s', resolution '%s', content-type: %s: %s",
+            layer_name,
+            wms_layer,
+            resolution,
+            response.headers["Content-Type"],
+            response.text,
+        )
+        return None
+    try:
+        return Image.open(BytesIO(response.content))
+    except Exception:  # pylint: disable=broad-exception-caught
+        if out is not None:
+            print(
+                "\n".join(
+                    [
+                        f"Unable to read legend image for layer '{layer_name}'-'{wms_layer}', resolution '{resolution}'",
+                        url,
+                        response.text,
+                    ]
+                ),
+                file=out,
+            )
+        _LOGGER.debug(
+            "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s",
+            layer_name,
+            wms_layer,
+            resolution,
+            response.text,
+            exc_info=True,
+        )
+        return None
+
+
+def _generate_legend_images(gene: TileGeneration, out: IO[str] | None = None) -> None:
     assert gene.config_file
     config = gene.get_config(gene.config_file)
     cache = config.config["caches"][gene.options.cache]
@@ -374,7 +479,7 @@ def _generate_legend_images(gene: TileGeneration) -> None:
             previous_hash = None
             for zoom, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]):
                 legends = []
-                for wmslayer in layer["layers"].split(","):
+                for wms_layer in layer["layers"].split(","):
                     url = (
                         layer["url"]
                         + "?"
@@ -383,100 +488,26 @@ def _generate_legend_images(gene: TileGeneration) -> None:
                                 "SERVICE": "WMS",
                                 "VERSION": layer.get("version", "1.0.0"),
                                 "REQUEST": "GetLegendGraphic",
-                                "LAYER": wmslayer,
+                                "LAYER": wms_layer,
                                 "FORMAT": layer["legend_mime"],
                                 "TRANSPARENT": "TRUE" if layer["legend_mime"] == "image/png" else "FALSE",
                                 "STYLE": layer["wmts_style"],
                                 "SCALE": resolution / 0.00028,
                             }
                         )
                     )
-                    _LOGGER.debug(
-                        "Get legend image for layer '%s'-'%s', resolution '%s': %s",
+                    legend_image = _get_legend_image(
                         layer_name,
-                        wmslayer,
+                        wms_layer,
                         resolution,
                         url,
+                        session,
+                        layer["legend_mime"].split("/")[0],
+                        out,
                     )
-                    try:
-                        response = session.get(url)
-                    except Exception as e:  # pylint: disable=broad-exception-caught
-                        print(
-                            "\n".join(
-                                [
-                                    f"Unable to get legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'",
-                                    url,
-                                    str(e),
-                                ]
-                            )
-                        )
-                        _LOGGER.debug(
-                            "Unable to get legend image for layer '%s'-'%s', resolution '%s'",
-                            layer_name,
-                            wmslayer,
-                            resolution,
-                            exc_info=True,
-                        )
-                        continue
-                    if response.status_code != 200:
-                        print(
-                            "\n".join(
-                                [
-                                    f"Unable to get legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'",
-                                    url,
-                                    f"status code: {response.status_code}: {response.reason}",
-                                    response.text,
-                                ]
-                            )
-                        )
-                        _LOGGER.debug(
-                            "Unable to get legend image for layer '%s'-'%s', resolution '%s': %s",
-                            layer_name,
-                            wmslayer,
-                            resolution,
-                            response.text,
-                        )
-                        continue
-                    if not response.headers["Content-Type"].startswith(layer["legend_mime"].split("/")[0]):
-                        print(
-                            "\n".join(
-                                [
-                                    f"Unable to get legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'",
-                                    url,
-                                    f"Content-Type: {response.headers['Content-Type']}",
-                                    response.text,
-                                ]
-                            )
-                        )
-                        _LOGGER.debug(
-                            "Unable to get legend image for layer '%s'-'%s', resolution '%s', content-type: %s: %s",
-                            layer_name,
-                            wmslayer,
-                            resolution,
-                            response.headers["Content-Type"],
-                            response.text,
-                        )
-                        continue
-                    try:
-                        legends.append(Image.open(BytesIO(response.content)))
-                    except Exception:  # pylint: disable=broad-exception-caught
-                        print(
-                            "\n".join(
-                                [
-                                    f"Unable to read legend image for layer '{layer_name}'-'{wmslayer}', resolution '{resolution}'",
-                                    url,
-                                    response.text,
-                                ]
-                            )
-                        )
-                        _LOGGER.debug(
-                            "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s",
-                            layer_name,
-                            wmslayer,
-                            resolution,
-                            response.text,
-                            exc_info=True,
-                        )
+                    if legend_image is not None:
+                        legends.append(legend_image)
+
                 width = max(1, max(i.size[0] for i in legends) if legends else 0)
                 height = max(1, sum(i.size[1] for i in legends) if legends else 0)
                 image = Image.new("RGBA", (width, height))

diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py
@@ -611,6 +611,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None:
             config_file=options.config or os.environ.get("TILEGENERATION_CONFIGFILE"),
             options=options,
             multi_thread=options.get_hash is None,
+            out=out,
         )
 
         if (

diff --git a/tilecloud_chain/schema.json b/tilecloud_chain/schema.json
@@ -1136,7 +1136,6 @@
             "--get-hash",
             "--generate-legend-images",
             "--get-bbox",
-            "--help",
             "--ignore-error",
             "--bbox",
             "--zoom",

diff --git a/tilecloud_chain/store/postgresql.py b/tilecloud_chain/store/postgresql.py
@@ -233,13 +233,13 @@ def _start_job(
             _LOGGER.info("Running the command `%s` using the function directly", display_command)
             main(final_command, out)
             _LOGGER.info("Successfully ran the command `%s` using the function directly", display_command)
-        except Exception:  # pylint: disable=broad-exception-caught
-            _LOGGER.exception("Error while running the command `%s`", display_command)
-            error = True
         except SystemExit as exception:
             if exception.code != 0:
                 _LOGGER.exception("Error while running the command `%s`", display_command)
                 error = True
+        except Exception:  # pylint: disable=broad-exception-caught
+            _LOGGER.exception("Error while running the command `%s`", display_command)
+            error = True
 
         with SessionMaker() as session:
             job = session.query(Job).where(Job.id == job_id).with_for_update(of=Job).one()