update and fix linting issues

Signed-off-by: kta-intel <[email protected]>

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Lint with Flake8
+name: Check code format
 
 on:
   pull_request:
@@ -21,11 +21,9 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: "3.8"
-    - name: Install dependencies
+    - name: Install linters
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-linters.txt 
-        pip install .
-    - name: Lint with flake8
-      run: |
-        flake8 --show-source
+        pip install -r linters-requirements.txt 
+    - name: Lint using built-in script
+      run: bash shell/lint.sh

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+---
+minimum_pre_commit_version: 3.3.3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+        files: \.py$
+      - id: requirements-txt-fixer
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-json
+        files: \.json$
+      - id: check-yaml
+        files: \.yaml$
+      - id: debug-statements
+      - id: pretty-format-json
+        args:
+          - --autofix
+        files: \.json$
+      - id: trailing-whitespace
+        files: \.py$
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.0
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place
+          - --remove-unused-variables
+          - --recursive
+          - --ignore-pass-statements

openfl_contrib/interface/aggregation_functions/custom_weighted_average.py

@@ -41,5 +41,7 @@ def call(self, local_tensors, *_) -> np.ndarray:
         tensors, weights = zip(*[(x.tensor, x.weight) for x in local_tensors])
 
         total_weight = sum(weights)
-        weighted_sum = np.sum([tensor * weight for tensor, weight in zip(tensors, weights)], axis=0)
-        return weighted_sum / total_weight
+        weighted_sum = np.sum(
+            [tensor * weight for tensor, weight in zip(tensors, weights)], axis=0
+        )
+        return weighted_sum / total_weight

openfl_contrib/pipelines/skc_pipeline.py

@@ -36,7 +36,7 @@ def forward(self, data, **kwargs):
             sparse_data: a flattened, sparse representation of the input tensor
             metadata: dictionary to store a list of meta information.
         """
-        metadata = {'int_list': list(data.shape)}
+        metadata = {"int_list": list(data.shape)}
         # sparsification
         data = data.astype(np.float32)
         flatten_data = data.flatten()
@@ -59,7 +59,7 @@ def backward(self, data, metadata, **kwargs):
             recovered_data: an numpy array with original shape.
         """
         data = data.astype(np.float32)
-        data_shape = metadata['int_list']
+        data_shape = metadata["int_list"]
         recovered_data = data.reshape(data_shape)
         return recovered_data
 
@@ -109,16 +109,15 @@ def forward(self, data, **kwargs):
         # clustering
         data = data.reshape((-1, 1))
         if data.shape[0] >= self.n_cluster:
-            k_means = cluster.KMeans(
-                n_clusters=self.n_cluster, n_init=self.n_cluster)
+            k_means = cluster.KMeans(n_clusters=self.n_cluster, n_init=self.n_cluster)
             k_means.fit(data)
             quantized_values = k_means.cluster_centers_.squeeze()
             indices = k_means.labels_
             quant_array = np.choose(indices, quantized_values)
         else:
             quant_array = data
         int_array, int2float_map = self._float_to_int(quant_array)
-        metadata = {'int_to_float': int2float_map}
+        metadata = {"int_to_float": int2float_map}
         int_array = int_array.reshape(-1)
         return int_array, metadata
 
@@ -135,7 +134,7 @@ def backward(self, data, metadata, **kwargs):
         """
         # convert back to float
         data = co.deepcopy(data)
-        int2float_map = metadata['int_to_float']
+        int2float_map = metadata["int_to_float"]
         for key in int2float_map:
             indices = data == key
             data[indices] = int2float_map[key]
@@ -221,6 +220,6 @@ def __init__(self, p_sparsity=0.1, n_clusters=6, **kwargs):
         transformers = [
             SparsityTransformer(self.p),
             KmeansTransformer(self.n_cluster),
-            GZIPTransformer()
+            GZIPTransformer(),
         ]
         super(SKCPipeline, self).__init__(transformers=transformers, **kwargs)

openfl_contrib_tutorials/ml_to_fl/central/requirements.txt

@@ -1,2 +1,2 @@
 torch==2.3.1
-torchvision==0.18.1
+torchvision==0.18.1

openfl_contrib_tutorials/ml_to_fl/federated/src/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
+# SPDX-License-Identifier: Apache-2.0

openfl_contrib_tutorials/ml_to_fl/federated/src/dataloader.py

@@ -70,7 +70,7 @@ def load_dataset(data_path, train_split_ratio=0.8, **kwargs):
     # Implement dataset loading logic here and return the appropriate data.
     # Replace the following placeholders with actual data loading code.
     dataset = MNISTDataset(
-        root=data_path, 
+        root=data_path,
         transform=Compose([Grayscale(num_output_channels=1), ToTensor()])
     )
     n_train = int(train_split_ratio * len(dataset))

openfl_contrib_workspace/torch_cnn_mnist_custom_weighted_average/requirements.txt

@@ -1,4 +1,4 @@
+tensorboard
 torch==2.3.0
 torchvision==0.18
-tensorboard
 wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability

openfl_contrib_workspace/torch_cnn_mnist_skc_compression/requirements.txt

@@ -1,4 +1,4 @@
+tensorboard
 torch==2.3.0
 torchvision==0.18
-tensorboard
 wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability

pyproject.toml

@@ -0,0 +1,7 @@
+[tool.black]
+line-length = 100
+
+[tool.isort]
+profile = "black"
+force_single_line = "False"
+line_length = 100

requirements-linters.txt

@@ -1,14 +1,4 @@
+black
 flake8
-flake8-broken-line
-flake8-bugbear
-flake8-builtins
-flake8-comprehensions
-flake8-copyright
-flake8-docstrings
-flake8-eradicate
-flake8-import-order
-flake8-import-single
-flake8-quotes
-flake8-use-fstring
-pep8-naming
-setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
+isort
+pre-commit

requirements-test.txt

@@ -1 +1 @@
-pytest==8.2.0
+pytest==8.2.0

setup.cfg

@@ -0,0 +1,19 @@
+[flake8]
+ignore =
+    # Conflicts with black
+    E203
+    # Line break occurred before a binary operator. Update by W504 Line
+    W503
+    # Allow "import torch.nn.functional as F"
+    N812
+
+per-file-ignores =
+    # Unused imports in __init__.py are OK
+    **/__init__.py:F401
+
+exclude = 
+    *_pb2*,
+
+max-line-length = 100
+
+copyright-check = True

setup.py

@@ -14,4 +14,4 @@
         'openfl @ git+https://github.com/securefederatedai/openfl.git@develop',
     ],
     python_requires='>=3.8, <3.12'
-)
+)

shell/format.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+base_dir=$(dirname $(dirname $0))
+
+# Run the pre-commit checks
+pre-commit run --all-files
+
+isort --sp "${base_dir}/pyproject.toml" openfl
+
+black --config "${base_dir}/pyproject.toml" openfl
+
+flake8 --config "${base_dir}/setup.cfg" openfl

shell/lint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+base_dir=$(dirname $(dirname $0))
+
+# Run the pre-commit checks
+pre-commit run --all-files
+
+isort --sp "${base_dir}/pyproject.toml" --check openfl
+
+black --config "${base_dir}/pyproject.toml" --check openfl
+
+flake8 --config "${base_dir}/setup.cfg" --show-source openfl

tests/github/utils.py

@@ -5,7 +5,7 @@
 import os
 from pathlib import Path
 import re
-import tarfile
+pass
 
 
 def create_collaborator(col, workspace_root, data_path, archive_name, fed_workspace):
@@ -80,4 +80,4 @@ def certify_aggregator(fqdn):
     check_call(['fx', 'aggregator', 'generate-cert-request', '--fqdn', fqdn])
 
     # Sign aggregator certificate
-    check_call(['fx', 'aggregator', 'certify', '--fqdn', fqdn, '--silent'])
+    check_call(['fx', 'aggregator', 'certify', '--fqdn', fqdn, '--silent'])

tests/openfl-contrib/interface/aggregation_functions/test_custom_weighted_average.py

@@ -30,7 +30,7 @@ def test_get_aggregated_tensor_weights(tensor_db):
     """Test that get_aggregated_tensor calculates correctly."""
 
     collaborator_weight_dict = {
-        'col1': 0.9, 
+        'col1': 0.9,
         'col2': 0.1
         }
 
@@ -45,4 +45,4 @@ def test_get_aggregated_tensor_weights(tensor_db):
         axis=0
     )
 
-    assert np.array_equal(agg_nparray, control_nparray)
+    assert np.array_equal(agg_nparray, control_nparray)