From dd1bce449fb91e4945206ad34abba2b30e3a14a2 Mon Sep 17 00:00:00 2001 From: Helen Theissen Date: Mon, 23 Sep 2024 16:07:49 +0100 Subject: [PATCH] Chore/multiple fixes ci precommit (#49) * fix: change pre-cmmit autoupdate scheule to monthly * fix: change the merge strategy for Changelog to Union * ci: fix downstream-ci-hpc workflow call * fix: add .envrc to .gitignore * ci: ignore pre-commit-config and readthedocs for changelog updates * fix: update precommit config * chore: update pre-commit * feat: add codeowners for workflow files * chore: update pyproject.toml all * fix: formatting * ci: add hpc config * docs: update changelog * docs: fix docstrings * ci: remove notebook pytest --------- Co-authored-by: Jesper Dramsch --- .gitattributes | 1 + .github/CODEOWNERS | 6 ++++ .github/ci-hpc-config.yml | 16 ++++++++++ .github/workflows/changelog-pr-update.yml | 3 ++ .github/workflows/ci.yml | 2 +- .gitignore | 1 + .pre-commit-config.yaml | 31 ++++++++++++++----- CHANGELOG.md | 6 ++++ pyproject.toml | 36 ++++------------------- src/anemoi/graphs/edges/builder.py | 18 +++++++++--- src/anemoi/graphs/generate/hexagonal.py | 15 ++++++++++ src/anemoi/graphs/generate/icosahedral.py | 11 +------ src/anemoi/graphs/nodes/attributes.py | 12 ++++++++ src/anemoi/graphs/nodes/builder.py | 2 ++ 14 files changed, 107 insertions(+), 53 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/CODEOWNERS create mode 100644 .github/ci-hpc-config.yml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a19ade0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +CHANGELOG.md merge=union diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..a2c619f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# CODEOWNERS file + +# Protect workflow files +/.github/ @theissenhelen @jesperdramsch @gmertes +/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes +/pyproject.toml @theissenhelen @jesperdramsch @gmertes diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..bbe6ef6 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,16 @@ +build: + modules: + - ninja + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/eccodes@develop + - ecmwf/eckit@develop + - ecmwf/odc@develop + python_dependencies: + - ecmwf/anemoi-utils@develop + - ecmwf/anemoi-datasets@develop + parallel: 64 + + pytest_cmd: | + python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml + python -m coverage report diff --git a/.github/workflows/changelog-pr-update.yml b/.github/workflows/changelog-pr-update.yml index 4bc51df..73cb1eb 100644 --- a/.github/workflows/changelog-pr-update.yml +++ b/.github/workflows/changelog-pr-update.yml @@ -5,6 +5,9 @@ on: branches: - main - develop + paths-ignore: + - .pre-commit-config.yaml + - .readthedocs.yaml jobs: Check-Changelog: name: Check Changelog Action diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9e9f91..1844abc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main with: anemoi-graphs: ecmwf/anemoi-graphs@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit diff --git a/.gitignore b/.gitignore index 1b49006..c610ac1 100644 --- a/.gitignore +++ b/.gitignore @@ -120,6 +120,7 @@ celerybeat.pid *.sage.py # Environments +.envrc .env .venv env/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c042b1f..f3c3962 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,12 @@ repos: - id: no-commit-to-branch # Prevent committing to main / master - id: check-added-large-files # Check for large files added to git - id: check-merge-conflict # Check for files that contain merge conflict +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 # Use the ref you want to point at + hooks: + - id: python-use-type-annotations # Check for missing type annotations + - id: python-check-blanket-noqa # Check for # noqa: all + - id: python-no-log-warn # Check for log.warn - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.8.0 hooks: @@ -34,7 +40,7 @@ repos: - --force-single-line-imports - --profile black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.6.4 hooks: - id: ruff # Next line if for documenation cod snippets @@ -45,7 +51,7 @@ repos: - --exit-non-zero-on-fix - --preview - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.9.1 + rev: v1.0.0 hooks: - id: sphinx-lint # For now, we use it. But it does not support a lot of sphinx features @@ -59,12 +65,21 @@ repos: hooks: - id: docconvert args: ["numpy"] -- repo: https://github.com/b8raoult/optional-dependencies-all - rev: "0.0.6" - hooks: - - id: optional-dependencies-all - args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "2.2.1" + rev: "2.2.3" hooks: - id: pyproject-fmt +- repo: https://github.com/jshwi/docsig # Check docstrings against function sig + rev: v0.60.1 + hooks: + - id: docsig + args: + - --ignore-no-params # Allow docstrings without parameters + - --check-dunders # Check dunder methods + - --check-overridden # Check overridden methods + - --check-protected # Check protected methods + - --check-class # Check class docstrings + - --disable=E113 # Disable empty docstrings + - --summary # Print a summary +ci: + autoupdate_schedule: monthly diff --git a/CHANGELOG.md b/CHANGELOG.md index 5beab08..2f572c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ Keep it human-readable, your future self will thank you! ## [Unreleased](https://github.com/ecmwf/anemoi-graphs/compare/0.3.0...HEAD) +### Added +- ci: hpc-config, CODEOWNERS (#49) + +### Changed +- ci: small fixes and updates pre-commit, downsteam-ci (#49) + ## [0.3.0 Anemoi-graphs, minor release](https://github.com/ecmwf/anemoi-graphs/compare/0.2.1...0.3.0) - 2024-09-03 ### Added diff --git a/pyproject.toml b/pyproject.toml index 5ccfe72..a6148ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,20 +10,13 @@ # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/ [build-system] -requires = [ - "setuptools>=60", - "setuptools-scm>=8", -] +requires = [ "setuptools>=60", "setuptools-scm>=8" ] [project] name = "anemoi-graphs" description = "A package to build graphs for data-driven forecasts." -keywords = [ - "ai", - "graphs", - "tools", -] +keywords = [ "ai", "graphs", "tools" ] license = { file = "LICENSE" } authors = [ @@ -46,9 +39,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] -dynamic = [ - "version", -] +dynamic = [ "version" ] dependencies = [ "anemoi-datasets[data]>=0.3.3", "anemoi-utils>=0.3.6", @@ -63,20 +54,8 @@ dependencies = [ "trimesh>=4.1", ] -optional-dependencies.all = [ -] -optional-dependencies.dev = [ - "nbsphinx", - "pandoc", - "pytest", - "pytest-mock", - "requests", - "sphinx", - "sphinx-argparse", - "sphinx-rtd-theme", - "termcolor", - "tomli", -] +optional-dependencies.all = [ ] +optional-dependencies.dev = [ "anemoi-graphs[docs,tests]" ] optional-dependencies.docs = [ "nbsphinx", @@ -89,10 +68,7 @@ optional-dependencies.docs = [ "tomli", ] -optional-dependencies.tests = [ - "pytest", - "pytest-mock", -] +optional-dependencies.tests = [ "pytest", "pytest-mock" ] urls.Documentation = "https://anemoi-graphs.readthedocs.io/" urls.Homepage = "https://github.com/ecmwf/anemoi-graphs/" diff --git a/src/anemoi/graphs/edges/builder.py b/src/anemoi/graphs/edges/builder.py index 389faec..7f34390 100644 --- a/src/anemoi/graphs/edges/builder.py +++ b/src/anemoi/graphs/edges/builder.py @@ -153,7 +153,7 @@ def __init__(self, source_name: str, target_name: str, num_nearest_neighbours: i assert num_nearest_neighbours > 0, "Number of nearest neighbours must be positive" self.num_nearest_neighbours = num_nearest_neighbours - def get_adjacency_matrix(self, source_nodes: np.ndarray, target_nodes: np.ndarray): + def get_adjacency_matrix(self, source_nodes: np.ndarray, target_nodes: np.ndarray) -> np.ndarray: """Compute the adjacency matrix for the KNN method. Parameters @@ -162,6 +162,11 @@ def get_adjacency_matrix(self, source_nodes: np.ndarray, target_nodes: np.ndarra The source nodes. target_nodes : np.ndarray The target nodes. + + Returns + ------- + np.ndarray + The adjacency matrix. """ assert self.num_nearest_neighbours is not None, "number of neighbors required for knn encoder" LOGGER.info( @@ -203,13 +208,13 @@ class CutOffEdges(BaseEdgeBuilder): Update the graph with the edges. """ - def __init__(self, source_name: str, target_name: str, cutoff_factor: float): + def __init__(self, source_name: str, target_name: str, cutoff_factor: float) -> None: super().__init__(source_name, target_name) assert isinstance(cutoff_factor, (int, float)), "Cutoff factor must be a float" assert cutoff_factor > 0, "Cutoff factor must be positive" self.cutoff_factor = cutoff_factor - def get_cutoff_radius(self, graph: HeteroData, mask_attr: torch.Tensor | None = None): + def get_cutoff_radius(self, graph: HeteroData, mask_attr: torch.Tensor | None = None) -> float: """Compute the cut-off radius. The cut-off radius is computed as the product of the target nodes @@ -238,7 +243,7 @@ def prepare_node_data(self, graph: HeteroData) -> tuple[NodeStorage, NodeStorage self.radius = self.get_cutoff_radius(graph) return super().prepare_node_data(graph) - def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStorage): + def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStorage) -> np.ndarray: """Get the adjacency matrix for the cut-off method. Parameters @@ -247,6 +252,11 @@ def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStor The source nodes. target_nodes : NodeStorage The target nodes. + + Returns + ------- + np.ndarray + The adjacency matrix. """ LOGGER.info( "Using CutOff-Edges (with radius = %.1f km) between %s and %s.", diff --git a/src/anemoi/graphs/generate/hexagonal.py b/src/anemoi/graphs/generate/hexagonal.py index 9e1bf29..2a9cfe3 100644 --- a/src/anemoi/graphs/generate/hexagonal.py +++ b/src/anemoi/graphs/generate/hexagonal.py @@ -62,6 +62,11 @@ def add_nodes_for_resolution( The H3 refinement level. It can be an integer from 0 to 15. area_kwargs: dict Additional arguments to pass to the get_nodes_at_resolution function. + + Returns + ------- + graph : networkx.Graph + The graph with the added nodes. """ nodes = get_nodes_at_resolution(resolution, **area_kwargs) @@ -173,6 +178,11 @@ def add_edges_to_children( depth_children : Optional[int], optional The number of resolution levels to consider for the connections of children. Defaults to 1, which includes connections up to the next resolution level, by default None + + Returns + ------- + graph : nx.Graph + graph with the added edges """ if depth_children is None: depth_children = len(refinement_levels) @@ -215,6 +225,11 @@ def add_edge( The H3 index of the tail of the edge. target_node_h3_idx : str The H3 index of the head of the edge. + + Returns + ------- + graph : networkx.Graph + The graph with the added edge. """ if not graph.has_node(source_node_h3_idx) or not graph.has_node(target_node_h3_idx): return graph diff --git a/src/anemoi/graphs/generate/icosahedral.py b/src/anemoi/graphs/generate/icosahedral.py index ed88621..357676a 100644 --- a/src/anemoi/graphs/generate/icosahedral.py +++ b/src/anemoi/graphs/generate/icosahedral.py @@ -22,8 +22,6 @@ def create_icosahedral_nodes( ---------- resolutions : list[int] Levels of mesh resolution to consider. - aoi_mask_builder : KNNAreaMaskBuilder - KNNAreaMaskBuilder with the cloud of points to limit the mesh area, by default None. Returns ------- @@ -86,10 +84,6 @@ def add_edges_to_nx_graph( Levels of mesh refinement levels to consider. x_hops : int, optional Number of hops between 2 nodes to consider them neighbours, by default 1. - aoi_mask_builder : KNNAreaMaskBuilder - NearestNeighbors with the cloud of points to limit the mesh area, by default None. - margin_radius_km : float, optional - Margin radius in km to consider when creating the processor mesh, by default 0.0. Returns ------- @@ -137,9 +131,6 @@ def get_neighbours_within_hops(tri_mesh: trimesh.Trimesh, x_hops: int) -> dict[i The mesh to consider. x_hops : int Number of hops between 2 nodes to consider them neighbours. - valid_nodes : list[int], optional - List of valid nodes to consider, by default None. It is useful to consider only a subset of the nodes to save - computation time. Returns ------- @@ -178,7 +169,7 @@ def add_neigbours_edges( A 2D array of shape (num_vertices, 2) with the planar coordinates of the mesh, in radians. node_idx : int The node considered. - neighbours : list[int] + neighbour_indices : list[int] The neighbours of the node. self_loops : bool, optional Whether is supported to add self-loops, by default False. diff --git a/src/anemoi/graphs/nodes/attributes.py b/src/anemoi/graphs/nodes/attributes.py index 11009d5..16a577d 100644 --- a/src/anemoi/graphs/nodes/attributes.py +++ b/src/anemoi/graphs/nodes/attributes.py @@ -43,6 +43,10 @@ def compute(self, graph: HeteroData, nodes_name: str, *args, **kwargs) -> torch. Graph. nodes_name : str Name of the nodes. + args : tuple + Additional arguments. + kwargs : dict + Additional keyword arguments. Returns ------- @@ -70,6 +74,10 @@ def get_raw_values(self, nodes: NodeStorage, *args, **kwargs) -> np.ndarray: ---------- nodes : NodeStorage Nodes of the graph. + args : tuple + Additional arguments. + kwargs : dict + Additional keyword arguments. Returns ------- @@ -111,6 +119,10 @@ def get_raw_values(self, nodes: NodeStorage, *args, **kwargs) -> np.ndarray: ---------- nodes : NodeStorage Nodes of the graph. + args : tuple + Additional arguments. + kwargs : dict + Additional keyword arguments. Returns ------- diff --git a/src/anemoi/graphs/nodes/builder.py b/src/anemoi/graphs/nodes/builder.py index 54753c4..12c818c 100644 --- a/src/anemoi/graphs/nodes/builder.py +++ b/src/anemoi/graphs/nodes/builder.py @@ -204,6 +204,8 @@ class IcosahedralNodes(BaseNodeBuilder, ABC): ---------- resolution : list[int] | int Refinement level of the mesh. + name : str + The name of the nodes. """ def __init__(