diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9ccba39..1c7f9fe 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Lint with Flake8 +name: Check code format on: pull_request: @@ -21,11 +21,9 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.8" - - name: Install dependencies + - name: Install linters run: | python -m pip install --upgrade pip - pip install -r requirements-linters.txt - pip install . - - name: Lint with flake8 - run: | - flake8 --show-source + pip install -r linters-requirements.txt + - name: Lint using built-in script + run: bash shell/lint.sh \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9b323cd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +--- +minimum_pre_commit_version: 3.3.3 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: end-of-file-fixer + files: \.py$ + - id: requirements-txt-fixer + - id: check-merge-conflict + - id: check-case-conflict + - id: check-json + files: \.json$ + - id: check-yaml + files: \.yaml$ + - id: debug-statements + - id: pretty-format-json + args: + - --autofix + files: \.json$ + - id: trailing-whitespace + files: \.py$ + - repo: https://github.com/PyCQA/autoflake + rev: v2.2.0 + hooks: + - id: autoflake + args: + - --in-place + - --remove-unused-variables + - --recursive + - --ignore-pass-statements \ No newline at end of file diff --git a/openfl_contrib/interface/aggregation_functions/custom_weighted_average.py b/openfl_contrib/interface/aggregation_functions/custom_weighted_average.py index deb214e..7e7a8ce 100644 --- a/openfl_contrib/interface/aggregation_functions/custom_weighted_average.py +++ b/openfl_contrib/interface/aggregation_functions/custom_weighted_average.py @@ -41,5 +41,7 @@ def call(self, local_tensors, *_) -> np.ndarray: tensors, weights = zip(*[(x.tensor, x.weight) for x in local_tensors]) total_weight = sum(weights) - weighted_sum = np.sum([tensor * weight for tensor, weight in zip(tensors, weights)], axis=0) - return weighted_sum / total_weight \ No newline at end of file + weighted_sum = np.sum( + [tensor * weight for tensor, weight in zip(tensors, weights)], axis=0 + ) + return weighted_sum / total_weight diff --git a/openfl_contrib/pipelines/skc_pipeline.py b/openfl_contrib/pipelines/skc_pipeline.py index b07f29b..fec7745 100644 --- a/openfl_contrib/pipelines/skc_pipeline.py +++ b/openfl_contrib/pipelines/skc_pipeline.py @@ -36,7 +36,7 @@ def forward(self, data, **kwargs): sparse_data: a flattened, sparse representation of the input tensor metadata: dictionary to store a list of meta information. """ - metadata = {'int_list': list(data.shape)} + metadata = {"int_list": list(data.shape)} # sparsification data = data.astype(np.float32) flatten_data = data.flatten() @@ -59,7 +59,7 @@ def backward(self, data, metadata, **kwargs): recovered_data: an numpy array with original shape. """ data = data.astype(np.float32) - data_shape = metadata['int_list'] + data_shape = metadata["int_list"] recovered_data = data.reshape(data_shape) return recovered_data @@ -109,8 +109,7 @@ def forward(self, data, **kwargs): # clustering data = data.reshape((-1, 1)) if data.shape[0] >= self.n_cluster: - k_means = cluster.KMeans( - n_clusters=self.n_cluster, n_init=self.n_cluster) + k_means = cluster.KMeans(n_clusters=self.n_cluster, n_init=self.n_cluster) k_means.fit(data) quantized_values = k_means.cluster_centers_.squeeze() indices = k_means.labels_ @@ -118,7 +117,7 @@ def forward(self, data, **kwargs): else: quant_array = data int_array, int2float_map = self._float_to_int(quant_array) - metadata = {'int_to_float': int2float_map} + metadata = {"int_to_float": int2float_map} int_array = int_array.reshape(-1) return int_array, metadata @@ -135,7 +134,7 @@ def backward(self, data, metadata, **kwargs): """ # convert back to float data = co.deepcopy(data) - int2float_map = metadata['int_to_float'] + int2float_map = metadata["int_to_float"] for key in int2float_map: indices = data == key data[indices] = int2float_map[key] @@ -221,6 +220,6 @@ def __init__(self, p_sparsity=0.1, n_clusters=6, **kwargs): transformers = [ SparsityTransformer(self.p), KmeansTransformer(self.n_cluster), - GZIPTransformer() + GZIPTransformer(), ] super(SKCPipeline, self).__init__(transformers=transformers, **kwargs) diff --git a/openfl_contrib_tutorials/ml_to_fl/central/requirements.txt b/openfl_contrib_tutorials/ml_to_fl/central/requirements.txt index c374025..62c8356 100644 --- a/openfl_contrib_tutorials/ml_to_fl/central/requirements.txt +++ b/openfl_contrib_tutorials/ml_to_fl/central/requirements.txt @@ -1,2 +1,2 @@ torch==2.3.1 -torchvision==0.18.1 \ No newline at end of file +torchvision==0.18.1 diff --git a/openfl_contrib_tutorials/ml_to_fl/federated/src/__init__.py b/openfl_contrib_tutorials/ml_to_fl/federated/src/__init__.py index 4582b4f..916f3a4 100644 --- a/openfl_contrib_tutorials/ml_to_fl/federated/src/__init__.py +++ b/openfl_contrib_tutorials/ml_to_fl/federated/src/__init__.py @@ -1,2 +1,2 @@ # Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file +# SPDX-License-Identifier: Apache-2.0 diff --git a/openfl_contrib_tutorials/ml_to_fl/federated/src/dataloader.py b/openfl_contrib_tutorials/ml_to_fl/federated/src/dataloader.py index aada401..7957688 100644 --- a/openfl_contrib_tutorials/ml_to_fl/federated/src/dataloader.py +++ b/openfl_contrib_tutorials/ml_to_fl/federated/src/dataloader.py @@ -70,7 +70,7 @@ def load_dataset(data_path, train_split_ratio=0.8, **kwargs): # Implement dataset loading logic here and return the appropriate data. # Replace the following placeholders with actual data loading code. dataset = MNISTDataset( - root=data_path, + root=data_path, transform=Compose([Grayscale(num_output_channels=1), ToTensor()]) ) n_train = int(train_split_ratio * len(dataset)) diff --git a/openfl_contrib_workspace/torch_cnn_mnist_custom_weighted_average/requirements.txt b/openfl_contrib_workspace/torch_cnn_mnist_custom_weighted_average/requirements.txt index 1132223..43d1b4b 100644 --- a/openfl_contrib_workspace/torch_cnn_mnist_custom_weighted_average/requirements.txt +++ b/openfl_contrib_workspace/torch_cnn_mnist_custom_weighted_average/requirements.txt @@ -1,4 +1,4 @@ +tensorboard torch==2.3.0 torchvision==0.18 -tensorboard wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/openfl_contrib_workspace/torch_cnn_mnist_skc_compression/requirements.txt b/openfl_contrib_workspace/torch_cnn_mnist_skc_compression/requirements.txt index 1132223..43d1b4b 100644 --- a/openfl_contrib_workspace/torch_cnn_mnist_skc_compression/requirements.txt +++ b/openfl_contrib_workspace/torch_cnn_mnist_skc_compression/requirements.txt @@ -1,4 +1,4 @@ +tensorboard torch==2.3.0 torchvision==0.18 -tensorboard wheel>=0.38.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b38d423 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.black] +line-length = 100 + +[tool.isort] +profile = "black" +force_single_line = "False" +line_length = 100 \ No newline at end of file diff --git a/requirements-linters.txt b/requirements-linters.txt index 9d2599f..f5ea6cb 100644 --- a/requirements-linters.txt +++ b/requirements-linters.txt @@ -1,14 +1,4 @@ +black flake8 -flake8-broken-line -flake8-bugbear -flake8-builtins -flake8-comprehensions -flake8-copyright -flake8-docstrings -flake8-eradicate -flake8-import-order -flake8-import-single -flake8-quotes -flake8-use-fstring -pep8-naming -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +isort +pre-commit diff --git a/requirements-test.txt b/requirements-test.txt index 060ed65..15d066a 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 \ No newline at end of file +pytest==8.2.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..886f95c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,19 @@ +[flake8] +ignore = + # Conflicts with black + E203 + # Line break occurred before a binary operator. Update by W504 Line + W503 + # Allow "import torch.nn.functional as F" + N812 + +per-file-ignores = + # Unused imports in __init__.py are OK + **/__init__.py:F401 + +exclude = + *_pb2*, + +max-line-length = 100 + +copyright-check = True diff --git a/setup.py b/setup.py index 5ea6363..295613a 100644 --- a/setup.py +++ b/setup.py @@ -14,4 +14,4 @@ 'openfl @ git+https://github.com/securefederatedai/openfl.git@develop', ], python_requires='>=3.8, <3.12' -) \ No newline at end of file +) diff --git a/shell/format.sh b/shell/format.sh new file mode 100755 index 0000000..5d8f2c1 --- /dev/null +++ b/shell/format.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -Eeuo pipefail + +base_dir=$(dirname $(dirname $0)) + +# Run the pre-commit checks +pre-commit run --all-files + +isort --sp "${base_dir}/pyproject.toml" openfl + +black --config "${base_dir}/pyproject.toml" openfl + +flake8 --config "${base_dir}/setup.cfg" openfl \ No newline at end of file diff --git a/shell/lint.sh b/shell/lint.sh new file mode 100644 index 0000000..f3cee9c --- /dev/null +++ b/shell/lint.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -Eeuo pipefail + +base_dir=$(dirname $(dirname $0)) + +# Run the pre-commit checks +pre-commit run --all-files + +isort --sp "${base_dir}/pyproject.toml" --check openfl + +black --config "${base_dir}/pyproject.toml" --check openfl + +flake8 --config "${base_dir}/setup.cfg" --show-source openfl \ No newline at end of file diff --git a/tests/github/utils.py b/tests/github/utils.py index 24550b8..2a376c9 100644 --- a/tests/github/utils.py +++ b/tests/github/utils.py @@ -5,7 +5,7 @@ import os from pathlib import Path import re -import tarfile +pass def create_collaborator(col, workspace_root, data_path, archive_name, fed_workspace): @@ -80,4 +80,4 @@ def certify_aggregator(fqdn): check_call(['fx', 'aggregator', 'generate-cert-request', '--fqdn', fqdn]) # Sign aggregator certificate - check_call(['fx', 'aggregator', 'certify', '--fqdn', fqdn, '--silent']) \ No newline at end of file + check_call(['fx', 'aggregator', 'certify', '--fqdn', fqdn, '--silent']) diff --git a/tests/openfl-contrib/interface/aggregation_functions/test_custom_weighted_average.py b/tests/openfl-contrib/interface/aggregation_functions/test_custom_weighted_average.py index be1d26c..91d90ca 100644 --- a/tests/openfl-contrib/interface/aggregation_functions/test_custom_weighted_average.py +++ b/tests/openfl-contrib/interface/aggregation_functions/test_custom_weighted_average.py @@ -30,7 +30,7 @@ def test_get_aggregated_tensor_weights(tensor_db): """Test that get_aggregated_tensor calculates correctly.""" collaborator_weight_dict = { - 'col1': 0.9, + 'col1': 0.9, 'col2': 0.1 } @@ -45,4 +45,4 @@ def test_get_aggregated_tensor_weights(tensor_db): axis=0 ) - assert np.array_equal(agg_nparray, control_nparray) \ No newline at end of file + assert np.array_equal(agg_nparray, control_nparray)