From 575cae03fe47ed762cdc4d4071c30f8a2bfdf366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 14 Nov 2024 12:18:19 +0100 Subject: [PATCH] Fix new Ruff issues --- tilecloud_chain/__init__.py | 28 ++++++++++++++++++++++------ tilecloud_chain/controller.py | 6 +++--- tilecloud_chain/copy_.py | 4 ++-- tilecloud_chain/cost.py | 2 +- tilecloud_chain/database_logger.py | 2 ++ tilecloud_chain/expiretiles.py | 2 +- tilecloud_chain/generate.py | 6 ++++-- tilecloud_chain/server.py | 3 ++- tilecloud_chain/store/postgresql.py | 2 +- tilecloud_chain/views/admin.py | 4 ++-- 10 files changed, 40 insertions(+), 19 deletions(-) diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index 65e57ccf8..b3fb58f01 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -25,7 +25,7 @@ from io import BytesIO from itertools import product from math import ceil, sqrt -from typing import IO, TYPE_CHECKING, Any, Optional, TextIO, TypedDict, Union, cast +from typing import IO, TYPE_CHECKING, Any, TextIO, TypedDict, cast import boto3 import botocore.client @@ -35,7 +35,7 @@ import psycopg2 import tilecloud.filter.error from azure.identity import DefaultAzureCredential -from azure.storage.blob import BlobServiceClient, ContainerClient, ContentSettings +from azure.storage.blob import BlobServiceClient, ContainerClient from c2cwsgiutils import sentry from PIL import Image from prometheus_client import Counter, Summary @@ -230,6 +230,7 @@ def __init__( ) def __call__(self, tile: Tile | None) -> Tile | None: + """Run the tile generation.""" if tile is None: return None @@ -302,6 +303,7 @@ def __init__(self, db: Any) -> None: self.db = db def __call__(self) -> None: + """Close the database.""" self.db.close() @@ -376,6 +378,7 @@ class TileFilter(logging.Filter): """A logging filter that adds request information to CEE logs.""" def filter(self, record: Any) -> bool: + """Add the request information to the log record.""" thread_id = threading.current_thread().native_id assert thread_id is not None log_info = LOGGING_CONTEXT.get(os.getpid(), {}).get(thread_id) @@ -1137,7 +1140,7 @@ def get_geoms( (extent[2], extent[1]), ) ) - for z, r in enumerate(config.config["grids"][layer["grid"]]["resolutions"]): + for z, _ in enumerate(config.config["grids"][layer["grid"]]["resolutions"]): geoms[z] = geom if self.options.near is None and self.options.geom: @@ -1145,7 +1148,7 @@ def get_geoms( with _GEOMS_GET_SUMMARY.labels(layer_name, host if host else self.options.host).time(): connection = psycopg2.connect(g["connection"]) cursor = connection.cursor() - sql = f"SELECT ST_AsBinary(geom) FROM (SELECT {g['sql']}) AS g" # nosec + sql = f"SELECT ST_AsBinary(geom) FROM (SELECT {g['sql']}) AS g" # nosec # noqa: S608 _LOGGER.info("Execute SQL: %s.", sql) cursor.execute(sql) geom_list = [loads_wkb(bytes(r[0])) for r in cursor.fetchall()] @@ -1454,6 +1457,7 @@ def __init__(self) -> None: self.lock = threading.Lock() def __call__(self, tile: Tile | None = None) -> Tile | None: + """Count the number of generated tile.""" with self.lock: self.nb += 1 return tile @@ -1468,6 +1472,7 @@ def __init__(self) -> None: self.lock = threading.Lock() def __call__(self, tile: Tile | None = None) -> Tile | None: + """Count the number of generated tile and measure the total generated size.""" if tile and tile.data: with self.lock: self.nb += 1 @@ -1499,8 +1504,9 @@ def __init__( self.count = count def __call__(self, tile: Tile) -> Tile | None: + """Drop the tile if the size and hash are the same as the specified ones.""" assert tile.data - if len(tile.data) != self.size or sha1(tile.data).hexdigest() != self.sha1code: # nosec + if len(tile.data) != self.size or sha1(tile.data).hexdigest() != self.sha1code: # noqa: S324 return tile else: if self.store is not None: @@ -1539,6 +1545,7 @@ def __init__( self.actions: dict[tuple[str, str], Callable[[Tile], Tile | None] | None] = {} def __call__(self, tile: Tile) -> Tile | None: + """Run the action.""" layer = tile.metadata["layer"] config_file = tile.metadata["config_file"] action = self.actions.get((config_file, layer)) @@ -1559,6 +1566,7 @@ def __init__(self, block: str, out: IO[str] | None) -> None: self.out = out def __call__(self, tile: Tile) -> Tile: + """Log the tile size and hash.""" ref = None try: assert tile.data @@ -1579,7 +1587,7 @@ def __call__(self, tile: Tile) -> Tile: f"""Tile: {tile.tilecoord} {tile.formated_metadata} {self.block}: size: {len(tile.data)} - hash: {sha1(tile.data).hexdigest()}""", # nosec + hash: {sha1(tile.data).hexdigest()}""", # noqa: E501 file=self.out, ) return tile @@ -1604,6 +1612,7 @@ def filter(self, tilecoord: TileCoord) -> bool: return nb % self.nb_process == self.process_nb def __call__(self, tile: Tile) -> Tile | None: + """Filter the tile.""" return tile if self.filter(tile.tilecoord) else None @@ -1636,6 +1645,7 @@ def filter_tilecoord( ).intersects(geoms[tilecoord.z]) def __call__(self, tile: Tile) -> Tile | None: + """Filter the tile on a geometry.""" return ( tile if self.filter_tilecoord(self.gene.get_tile_config(tile), tile.tilecoord, tile.metadata["layer"]) @@ -1655,6 +1665,7 @@ def __init__(self, gene: TileGeneration) -> None: self.gene = gene def __call__(self, tile: Tile) -> Tile | None: + """Filter the enpty tile.""" config = self.gene.get_tile_config(tile) if not tile or not tile.data: _LOGGER.error( @@ -1712,6 +1723,7 @@ def __init__(self, config: tilecloud_chain.configuration.ProcessCommand, options self.options = options def __call__(self, tile: Tile) -> Tile | None: + """Process the tile.""" if tile and tile.data: fd_in, name_in = tempfile.mkstemp() with open(name_in, "wb") as file_in: @@ -1781,6 +1793,7 @@ def __init__(self, tiles_file: str): self.tiles_file = open(tiles_file, encoding="utf-8") # pylint: disable=consider-using-with def list(self) -> Iterator[Tile]: + """List the tiles.""" while True: line = self.tiles_file.readline() if not line: @@ -1804,12 +1817,15 @@ def list(self) -> Iterator[Tile]: ) def get_one(self, tile: Tile) -> Tile | None: + """Get the tile.""" raise NotImplementedError() def put_one(self, tile: Tile) -> Tile: + """Put the tile.""" raise NotImplementedError() def delete_one(self, tile: Tile) -> Tile: + """Delete the tile.""" raise NotImplementedError() diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py index f32bd32de..25fef274a 100644 --- a/tilecloud_chain/controller.py +++ b/tilecloud_chain/controller.py @@ -98,7 +98,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: except SystemExit: raise - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 _LOGGER.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise @@ -348,7 +348,7 @@ def _fill_legend( if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends: gene.layer_legends[layer_name] = [] legends = gene.layer_legends[layer_name] - for zoom, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]): + for _, resolution in enumerate(config.config["grids"][layer["grid"]]["resolutions"]): new_legend = legend_image_metadata.get(layer_name, {}).get(resolution) if new_legend is not None: @@ -412,7 +412,7 @@ def _generate_legend_images(gene: TileGeneration) -> None: string_io = BytesIO() image.save(string_io, FORMAT_BY_CONTENT_TYPE[layer["legend_mime"]]) result = string_io.getvalue() - new_hash = sha1(result).hexdigest() # nosec + new_hash = sha1(result).hexdigest() # nosec # noqa: S303 if new_hash != previous_hash: previous_hash = new_hash _send( diff --git a/tilecloud_chain/copy_.py b/tilecloud_chain/copy_.py index 573211b29..29d41071c 100644 --- a/tilecloud_chain/copy_.py +++ b/tilecloud_chain/copy_.py @@ -116,7 +116,7 @@ def main() -> None: copy.copy(options, gene, layer, options.source, options.dest, "copy") except SystemExit: raise - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 logger.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise @@ -151,6 +151,6 @@ def process() -> None: copy.copy(options, gene, layer, options.cache, options.cache, "process") except SystemExit: raise - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 logger.exception("Exit with exception") sys.exit(1) diff --git a/tilecloud_chain/cost.py b/tilecloud_chain/cost.py index ae1f7f0b4..68f86b5d0 100644 --- a/tilecloud_chain/cost.py +++ b/tilecloud_chain/cost.py @@ -90,7 +90,7 @@ def main() -> None: # gene.config['cost'].get("request_per_layers", configuration.REQUESTS_PER_LAYERS_DEFAULT) * tile_size) except SystemExit: raise - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 logger.exception("Exit with exception") sys.exit(1) diff --git a/tilecloud_chain/database_logger.py b/tilecloud_chain/database_logger.py index f186c01aa..d6ade67cd 100644 --- a/tilecloud_chain/database_logger.py +++ b/tilecloud_chain/database_logger.py @@ -99,6 +99,7 @@ def __init__(self, config: tilecloud_chain.configuration.Logging, daemon: bool) (self.run,) = cursor.fetchone() def __call__(self, tile: Tile) -> Tile: + """Log the generated tiles in a database.""" tile.metadata["run"] = self.run return tile @@ -107,6 +108,7 @@ class DatabaseLogger(DatabaseLoggerCommon): """Log the generated tiles in a database.""" def __call__(self, tile: Tile) -> Tile: + """Log the generated tiles in a database.""" if tile is None: _LOGGER.warning("The tile is None") return None diff --git a/tilecloud_chain/expiretiles.py b/tilecloud_chain/expiretiles.py index a522cfc09..414d876a9 100644 --- a/tilecloud_chain/expiretiles.py +++ b/tilecloud_chain/expiretiles.py @@ -182,6 +182,6 @@ def main() -> None: print("Import successful") except SystemExit: raise - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 logger.exception("Exit with exception") sys.exit(1) diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py index b7dd6dbd3..3d9f5cbf2 100644 --- a/tilecloud_chain/generate.py +++ b/tilecloud_chain/generate.py @@ -51,6 +51,7 @@ def __init__(self, gene: TileGeneration): self.gene = gene def __call__(self, tile: Tile) -> Tile: + """Add logs tile context.""" tilecloud_chain.LOGGING_CONTEXT.setdefault(os.getpid(), {})[threading.current_thread().native_id] = { # type: ignore "host": tile.metadata.get("host"), "layer": tile.metadata.get("layer"), @@ -435,6 +436,7 @@ def __init__(self, gene: Generate): self.gene = gene def __call__(self, config_file: str, layer_name: str) -> TileStore | None: + """Get the tilestore based on the layername config file any layer type.""" config = self.gene._gene.get_config(config_file) layer = config.config["layers"][layer_name] if layer["type"] == "wms": @@ -442,7 +444,7 @@ def __call__(self, config_file: str, layer_name: str) -> TileStore | None: if "STYLES" not in params: params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(",")) if layer.get("generate_salt", False): - params["SALT"] = str(random.randint(0, 999999)) # nosec + params["SALT"] = str(random.randint(0, 999999)) # nosec # noqa: S311 # Get the metatile image from the WMS server return TimedTileStoreWrapper( @@ -642,7 +644,7 @@ def main(args: list[str] | None = None, out: IO[str] | None = None) -> None: gene.close() except SystemExit: raise - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 _LOGGER.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise diff --git a/tilecloud_chain/server.py b/tilecloud_chain/server.py index 17d831abc..bfc264157 100644 --- a/tilecloud_chain/server.py +++ b/tilecloud_chain/server.py @@ -643,7 +643,7 @@ def forward( headers["Cache-Control"] = "no-cache" headers["Pragma"] = "no-cache" - response = requests.get(url, headers=headers) # nosec + response = requests.get(url, headers=headers) if response.status_code == 200: response_headers = dict(response.headers) if no_cache: @@ -802,6 +802,7 @@ def response( return request.response def get_host(self, **kwargs: Any) -> str: + """Get the host used in Prometheus stats and in the JSON logs.""" request: pyramid.request.Request = kwargs["request"] assert isinstance(request.host, str) return request.host diff --git a/tilecloud_chain/store/postgresql.py b/tilecloud_chain/store/postgresql.py index 616cc2bb4..b4ef83a5b 100644 --- a/tilecloud_chain/store/postgresql.py +++ b/tilecloud_chain/store/postgresql.py @@ -248,7 +248,7 @@ def _start_job( _LOGGER.info("Run the command `%s`", display_command) - completed_process = subprocess.run( # nosec # pylint: disable=subprocess-run-check + completed_process = subprocess.run( # nosec # pylint: disable=subprocess-run-check # noqa: S603 final_command, capture_output=True, env=env, diff --git a/tilecloud_chain/views/admin.py b/tilecloud_chain/views/admin.py index 1f591cf6b..3fbfbc965 100644 --- a/tilecloud_chain/views/admin.py +++ b/tilecloud_chain/views/admin.py @@ -184,7 +184,7 @@ def run(self) -> pyramid.response.Response: proc.join() return return_dict - completed_process = subprocess.run( # nosec # pylint: disable=subprocess-run-check + completed_process = subprocess.run( # nosec # pylint: disable=subprocess-run-check # noqa: S603 final_command, capture_output=True, env=env, @@ -341,7 +341,7 @@ def _parse_stdout(stdout: str) -> list[str]: full_message = json_message["full_message"].replace("\n", "
") msg += f"
{full_message}" stdout_parsed.append(msg) - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 stdout_parsed.append(line) return stdout_parsed