Chore/multiple fixes ci precommit (#49)

* fix: change pre-cmmit autoupdate scheule to monthly * fix: change the merge strategy for Changelog to Union * ci: fix downstream-ci-hpc workflow call * fix: add .envrc to .gitignore * ci: ignore pre-commit-config and readthedocs for changelog updates * fix: update precommit config * chore: update pre-commit * feat: add codeowners for workflow files * chore: update pyproject.toml all * fix: formatting * ci: add hpc config * docs: update changelog * docs: fix docstrings * ci: remove notebook pytest --------- Co-authored-by: Jesper Dramsch <[email protected]>
ecmwf · Sep 23, 2024 · dd1bce4 · dd1bce4
1 parent 7cbad1e
commit dd1bce4
Show file tree

Hide file tree

Showing 14 changed files with 107 additions and 53 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+CHANGELOG.md merge=union
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,6 @@
+# CODEOWNERS file
+
+# Protect workflow files
+/.github/ @theissenhelen @jesperdramsch @gmertes
+/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes
+/pyproject.toml @theissenhelen @jesperdramsch @gmertes
diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml
@@ -0,0 +1,16 @@
+build:
+  modules:
+    - ninja
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/odc@develop
+  python_dependencies:
+    - ecmwf/anemoi-utils@develop
+    - ecmwf/anemoi-datasets@develop
+  parallel: 64
+
+  pytest_cmd: |
+    python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml
+    python -m coverage report
diff --git a/.github/workflows/changelog-pr-update.yml b/.github/workflows/changelog-pr-update.yml
@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - develop
+    paths-ignore:
+      - .pre-commit-config.yaml
+      - .readthedocs.yaml
 jobs:
   Check-Changelog:
     name: Check Changelog Action

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -46,7 +46,7 @@ jobs:
   downstream-ci-hpc:
     name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       anemoi-graphs: ecmwf/anemoi-graphs@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit
diff --git a/.gitignore b/.gitignore
@@ -120,6 +120,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.envrc
 .env
 .venv
 env/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -20,6 +20,12 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
   rev: 24.8.0
   hooks:
@@ -34,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.3
+  rev: v0.6.4
   hooks:
   - id: ruff
     # Next line if for documenation cod snippets
@@ -45,7 +51,7 @@ repos:
     - --exit-non-zero-on-fix
     - --preview
 - repo: https://github.com/sphinx-contrib/sphinx-lint
-  rev: v0.9.1
+  rev: v1.0.0
   hooks:
   - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
@@ -59,12 +65,21 @@ repos:
   hooks:
   - id: docconvert
     args: ["numpy"]
-- repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.6"
-  hooks:
-  - id: optional-dependencies-all
-    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.2.1"
+  rev: "2.2.3"
   hooks:
   - id: pyproject-fmt
+-   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
+    rev: v0.60.1
+    hooks:
+    -   id: docsig
+        args:
+        - --ignore-no-params # Allow docstrings without parameters
+        - --check-dunders    # Check dunder methods
+        - --check-overridden # Check overridden methods
+        - --check-protected  # Check protected methods
+        - --check-class      # Check class docstrings
+        - --disable=E113     # Disable empty docstrings
+        - --summary          # Print a summary
+ci:
+  autoupdate_schedule: monthly
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,12 @@ Keep it human-readable, your future self will thank you!
 
 ## [Unreleased](https://github.com/ecmwf/anemoi-graphs/compare/0.3.0...HEAD)
 
+### Added
+- ci: hpc-config, CODEOWNERS (#49)
+
+### Changed
+- ci: small fixes and updates pre-commit, downsteam-ci (#49)
+
 ## [0.3.0 Anemoi-graphs, minor release](https://github.com/ecmwf/anemoi-graphs/compare/0.2.1...0.3.0) - 2024-09-03
 
 ### Added

diff --git a/pyproject.toml b/pyproject.toml
@@ -10,20 +10,13 @@
 # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
 
 [build-system]
-requires = [
-  "setuptools>=60",
-  "setuptools-scm>=8",
-]
+requires = [ "setuptools>=60", "setuptools-scm>=8" ]
 
 [project]
 name = "anemoi-graphs"
 
 description = "A package to build graphs for data-driven forecasts."
-keywords = [
-  "ai",
-  "graphs",
-  "tools",
-]
+keywords = [ "ai", "graphs", "tools" ]
 
 license = { file = "LICENSE" }
 authors = [
@@ -46,9 +39,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 
-dynamic = [
-  "version",
-]
+dynamic = [ "version" ]
 dependencies = [
   "anemoi-datasets[data]>=0.3.3",
   "anemoi-utils>=0.3.6",
@@ -63,20 +54,8 @@ dependencies = [
   "trimesh>=4.1",
 ]
 
-optional-dependencies.all = [
-]
-optional-dependencies.dev = [
-  "nbsphinx",
-  "pandoc",
-  "pytest",
-  "pytest-mock",
-  "requests",
-  "sphinx",
-  "sphinx-argparse",
-  "sphinx-rtd-theme",
-  "termcolor",
-  "tomli",
-]
+optional-dependencies.all = [  ]
+optional-dependencies.dev = [ "anemoi-graphs[docs,tests]" ]
 
 optional-dependencies.docs = [
   "nbsphinx",
@@ -89,10 +68,7 @@ optional-dependencies.docs = [
   "tomli",
 ]
 
-optional-dependencies.tests = [
-  "pytest",
-  "pytest-mock",
-]
+optional-dependencies.tests = [ "pytest", "pytest-mock" ]
 
 urls.Documentation = "https://anemoi-graphs.readthedocs.io/"
 urls.Homepage = "https://github.com/ecmwf/anemoi-graphs/"

diff --git a/src/anemoi/graphs/edges/builder.py b/src/anemoi/graphs/edges/builder.py
@@ -153,7 +153,7 @@ def __init__(self, source_name: str, target_name: str, num_nearest_neighbours: i
         assert num_nearest_neighbours > 0, "Number of nearest neighbours must be positive"
         self.num_nearest_neighbours = num_nearest_neighbours
 
-    def get_adjacency_matrix(self, source_nodes: np.ndarray, target_nodes: np.ndarray):
+    def get_adjacency_matrix(self, source_nodes: np.ndarray, target_nodes: np.ndarray) -> np.ndarray:
         """Compute the adjacency matrix for the KNN method.
 
         Parameters
@@ -162,6 +162,11 @@ def get_adjacency_matrix(self, source_nodes: np.ndarray, target_nodes: np.ndarra
             The source nodes.
         target_nodes : np.ndarray
             The target nodes.
+
+        Returns
+        -------
+        np.ndarray
+            The adjacency matrix.
         """
         assert self.num_nearest_neighbours is not None, "number of neighbors required for knn encoder"
         LOGGER.info(
@@ -203,13 +208,13 @@ class CutOffEdges(BaseEdgeBuilder):
         Update the graph with the edges.
     """
 
-    def __init__(self, source_name: str, target_name: str, cutoff_factor: float):
+    def __init__(self, source_name: str, target_name: str, cutoff_factor: float) -> None:
         super().__init__(source_name, target_name)
         assert isinstance(cutoff_factor, (int, float)), "Cutoff factor must be a float"
         assert cutoff_factor > 0, "Cutoff factor must be positive"
         self.cutoff_factor = cutoff_factor
 
-    def get_cutoff_radius(self, graph: HeteroData, mask_attr: torch.Tensor | None = None):
+    def get_cutoff_radius(self, graph: HeteroData, mask_attr: torch.Tensor | None = None) -> float:
         """Compute the cut-off radius.
 
         The cut-off radius is computed as the product of the target nodes
@@ -238,7 +243,7 @@ def prepare_node_data(self, graph: HeteroData) -> tuple[NodeStorage, NodeStorage
         self.radius = self.get_cutoff_radius(graph)
         return super().prepare_node_data(graph)
 
-    def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStorage):
+    def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStorage) -> np.ndarray:
         """Get the adjacency matrix for the cut-off method.
 
         Parameters
@@ -247,6 +252,11 @@ def get_adjacency_matrix(self, source_nodes: NodeStorage, target_nodes: NodeStor
             The source nodes.
         target_nodes : NodeStorage
             The target nodes.
+
+        Returns
+        -------
+        np.ndarray
+            The adjacency matrix.
         """
         LOGGER.info(
             "Using CutOff-Edges (with radius = %.1f km) between %s and %s.",

diff --git a/src/anemoi/graphs/generate/hexagonal.py b/src/anemoi/graphs/generate/hexagonal.py
@@ -62,6 +62,11 @@ def add_nodes_for_resolution(
         The H3 refinement level. It can be an integer from 0 to 15.
     area_kwargs: dict
         Additional arguments to pass to the get_nodes_at_resolution function.
+
+    Returns
+    -------
+    graph : networkx.Graph
+        The graph with the added nodes.
     """
 
     nodes = get_nodes_at_resolution(resolution, **area_kwargs)
@@ -173,6 +178,11 @@ def add_edges_to_children(
     depth_children : Optional[int], optional
         The number of resolution levels to consider for the connections of children. Defaults to 1, which includes
         connections up to the next resolution level, by default None
+
+    Returns
+    -------
+    graph : nx.Graph
+        graph with the added edges
     """
     if depth_children is None:
         depth_children = len(refinement_levels)
@@ -215,6 +225,11 @@ def add_edge(
         The H3 index of the tail of the edge.
     target_node_h3_idx : str
         The H3 index of the head of the edge.
+
+    Returns
+    -------
+    graph : networkx.Graph
+        The graph with the added edge.
     """
     if not graph.has_node(source_node_h3_idx) or not graph.has_node(target_node_h3_idx):
         return graph

diff --git a/src/anemoi/graphs/generate/icosahedral.py b/src/anemoi/graphs/generate/icosahedral.py
@@ -22,8 +22,6 @@ def create_icosahedral_nodes(
     ----------
     resolutions : list[int]
         Levels of mesh resolution to consider.
-    aoi_mask_builder : KNNAreaMaskBuilder
-        KNNAreaMaskBuilder with the cloud of points to limit the mesh area, by default None.
 
     Returns
     -------
@@ -86,10 +84,6 @@ def add_edges_to_nx_graph(
         Levels of mesh refinement levels to consider.
     x_hops : int, optional
         Number of hops between 2 nodes to consider them neighbours, by default 1.
-    aoi_mask_builder : KNNAreaMaskBuilder
-        NearestNeighbors with the cloud of points to limit the mesh area, by default None.
-    margin_radius_km : float, optional
-        Margin radius in km to consider when creating the processor mesh, by default 0.0.
 
     Returns
     -------
@@ -137,9 +131,6 @@ def get_neighbours_within_hops(tri_mesh: trimesh.Trimesh, x_hops: int) -> dict[i
         The mesh to consider.
     x_hops : int
         Number of hops between 2 nodes to consider them neighbours.
-    valid_nodes : list[int], optional
-        List of valid nodes to consider, by default None. It is useful to consider only a subset of the nodes to save
-        computation time.
 
     Returns
     -------
@@ -178,7 +169,7 @@ def add_neigbours_edges(
         A 2D array of shape (num_vertices, 2) with the planar coordinates of the mesh, in radians.
     node_idx : int
         The node considered.
-    neighbours : list[int]
+    neighbour_indices : list[int]
         The neighbours of the node.
     self_loops : bool, optional
         Whether is supported to add self-loops, by default False.

diff --git a/src/anemoi/graphs/nodes/attributes.py b/src/anemoi/graphs/nodes/attributes.py
@@ -43,6 +43,10 @@ def compute(self, graph: HeteroData, nodes_name: str, *args, **kwargs) -> torch.
             Graph.
         nodes_name : str
             Name of the nodes.
+        args : tuple
+            Additional arguments.
+        kwargs : dict
+            Additional keyword arguments.
 
         Returns
         -------
@@ -70,6 +74,10 @@ def get_raw_values(self, nodes: NodeStorage, *args, **kwargs) -> np.ndarray:
         ----------
         nodes : NodeStorage
             Nodes of the graph.
+        args : tuple
+            Additional arguments.
+        kwargs : dict
+            Additional keyword arguments.
 
         Returns
         -------
@@ -111,6 +119,10 @@ def get_raw_values(self, nodes: NodeStorage, *args, **kwargs) -> np.ndarray:
         ----------
         nodes : NodeStorage
             Nodes of the graph.
+        args : tuple
+            Additional arguments.
+        kwargs : dict
+            Additional keyword arguments.
 
         Returns
         -------

diff --git a/src/anemoi/graphs/nodes/builder.py b/src/anemoi/graphs/nodes/builder.py
@@ -204,6 +204,8 @@ class IcosahedralNodes(BaseNodeBuilder, ABC):
     ----------
     resolution : list[int] | int
         Refinement level of the mesh.
+    name : str
+        The name of the nodes.
     """
 
     def __init__(