From 8e2aaea83edd88e4b31ede145d5d9adcbd687257 Mon Sep 17 00:00:00 2001 From: Johanna England Date: Wed, 3 Jul 2024 15:47:22 +0200 Subject: [PATCH 1/6] Switch to MegaLinter for linting --- .github/workflows/linter.yml | 78 ++++++++++++++---------------------- .gitignore | 1 + .mega-linter.yml | 16 ++++++++ pyproject.toml | 3 +- 4 files changed, 50 insertions(+), 48 deletions(-) create mode 100644 .mega-linter.yml diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index e351306a02..4b40da7ba2 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,61 +1,45 @@ --- -################################# -################################# -## Super Linter GitHub Actions ## -################################# -################################# -name: Lint Code Base - -# -# Documentation: -# https://help.github.com/en/articles/workflow-syntax-for-github-actions -# - -############################# -# Start the job on all push # -############################# +# MegaLinter GitHub Action configuration file +# More info at https://megalinter.io +name: MegaLinter on: push: - #branches-ignore: [master, main] - # Remove the line above to run when pushing to master + branches: master pull_request: - branches: [master, main] -############### -# Set the Job # -############### jobs: - build: - # Name the Job - name: Lint Code Base - # Set the agent to run on + megalinter: + name: MegaLinter runs-on: ubuntu-latest - - ################## - # Load all steps # - ################## + permissions: + # Give the linter write permission to comment on PRs (if PR is not from fork) + issues: write + pull-requests: write steps: - ########################## - # Checkout the code base # - ########################## + # Git Checkout - name: Checkout Code uses: actions/checkout@v4 with: - # Full git history is needed to get a proper list of changed files within `super-linter` + token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 - - ################################ - # Run Linter against code base # - ################################ - - name: Lint Code Base - uses: github/super-linter/slim@v6 + # MegaLinter + - name: MegaLinter + id: ml + # You can override MegaLinter flavor used to have faster performances + # More info at https://megalinter.io/flavors/ + uses: oxsecurity/megalinter/flavors/python@v8 env: - VALIDATE_ALL_CODEBASE: false - DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_PYTHON_BLACK: true - VALIDATE_PYTHON_FLAKE8: true - LINTER_RULES_PATH: / - PYTHON_BLACK_CONFIG_FILE: pyproject.toml - PYTHON_FLAKE8_CONFIG_FILE: tox.ini - FILTER_REGEX_EXCLUDE: .*python/nav/smidumps/.* + # Validate whole codebase on pushes and only changes on pull requests + # Change this to VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'push' }} + VALIDATE_ALL_CODEBASE: true + + # Upload MegaLinter artifacts + - name: Archive production artifacts + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log diff --git a/.gitignore b/.gitignore index 573624ff64..3279c76922 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .* !.gitkeep !.github/ +!.mega-linter.yml *.bak *.iml # JetBrains (PyCharm/IDEA) project files *.rrd diff --git a/.mega-linter.yml b/.mega-linter.yml new file mode 100644 index 0000000000..e036297fc3 --- /dev/null +++ b/.mega-linter.yml @@ -0,0 +1,16 @@ +# All available variables are described in documentation +# https://megalinter.io/configuration/ + +LINTER_RULES_PATH: . + +ENABLE_LINTERS: + - PYTHON_BLACK + - PYTHON_FLAKE8 + +# Make workflow fail even on non blocking errors +FORMATTERS_DISABLE_ERRORS: false + +# Tell the linter the location of the configuration file (pyproject.toml) +LINTER_RULES_PATH: / +PYTHON_BLACK_CONFIG_FILE: pyproject.toml +PYTHON_FLAKE8_CONFIG_FILE: tox.ini diff --git a/pyproject.toml b/pyproject.toml index 8aed6e66bf..a4e403b140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,8 @@ where = ["python"] [tool.black] skip-string-normalization=true line-length = 88 -exclude = ''' +# Exclude files even when passed directly as argument (for MegaLinter) +force-exclude = ''' ( /( \.eggs # exclude a few common directories in the From 95f3804129ccf254ee1e5d49736d6802793aea8c Mon Sep 17 00:00:00 2001 From: Johanna England Date: Thu, 22 Aug 2024 16:23:32 +0200 Subject: [PATCH 2/6] Rerun black due to new version 24.8.0 --- tests/functional/room_test.py | 1 + tests/integration/eventengine/boxdown_test.py | 1 + tests/integration/smidump_test.py | 1 + tests/integration/statemondb_test.py | 1 + tests/unittests/arnold/arnold_memo_test.py | 1 + tests/unittests/arnold/arnold_snmp_test.py | 1 + tests/unittests/arnold/arnold_test.py | 1 + tests/unittests/business/business_test.py | 1 + tests/unittests/eventengine/severity_test.py | 2 +- tests/unittests/ipdevpoll/plugins_paloaltoarp_test.py | 7 ++++--- tests/unittests/ipdevpoll/timestamps_test.py | 1 + tests/unittests/logs_test.py | 1 + tests/unittests/metrics/thresholds_test.py | 1 + tests/unittests/portadmin/portadmin_util_test.py | 1 + tests/unittests/report/util_test.py | 1 + tools/eventgenerators/moduleevent.py | 4 ++-- 16 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/functional/room_test.py b/tests/functional/room_test.py index 971a39c60d..712d1cfb72 100644 --- a/tests/functional/room_test.py +++ b/tests/functional/room_test.py @@ -1,4 +1,5 @@ """Selenium tests for room views""" + import os from selenium.webdriver.common.by import By diff --git a/tests/integration/eventengine/boxdown_test.py b/tests/integration/eventengine/boxdown_test.py index a1040356f2..4c0141c6c0 100644 --- a/tests/integration/eventengine/boxdown_test.py +++ b/tests/integration/eventengine/boxdown_test.py @@ -1,4 +1,5 @@ """Black-box integration tests for the apparent proper processing of boxState events""" + import os import pytest diff --git a/tests/integration/smidump_test.py b/tests/integration/smidump_test.py index e0aa96af48..7fffeb7897 100644 --- a/tests/integration/smidump_test.py +++ b/tests/integration/smidump_test.py @@ -1,4 +1,5 @@ """Regression tests to validate smidump file contents""" + import os import pynetsnmp.netsnmp diff --git a/tests/integration/statemondb_test.py b/tests/integration/statemondb_test.py index 69db6ff323..dcf83446ca 100644 --- a/tests/integration/statemondb_test.py +++ b/tests/integration/statemondb_test.py @@ -1,4 +1,5 @@ """Some simple integration tests for the legacy statemon system""" + from nav.statemon import db diff --git a/tests/unittests/arnold/arnold_memo_test.py b/tests/unittests/arnold/arnold_memo_test.py index 1ba71fbe83..311ca2d07d 100644 --- a/tests/unittests/arnold/arnold_memo_test.py +++ b/tests/unittests/arnold/arnold_memo_test.py @@ -1,4 +1,5 @@ """Testcase for Arnold's memoisation class used for fetching info from files""" + import unittest from mock import Mock, patch from nav.arnold import Memo diff --git a/tests/unittests/arnold/arnold_snmp_test.py b/tests/unittests/arnold/arnold_snmp_test.py index 506234a00f..25ffcab18e 100644 --- a/tests/unittests/arnold/arnold_snmp_test.py +++ b/tests/unittests/arnold/arnold_snmp_test.py @@ -1,4 +1,5 @@ """Tests for Arnold using snmp objects""" + from nav.arnold import change_port_status from unittest.mock import Mock, patch import unittest diff --git a/tests/unittests/arnold/arnold_test.py b/tests/unittests/arnold/arnold_test.py index 9e5a638793..f83ca57f86 100644 --- a/tests/unittests/arnold/arnold_test.py +++ b/tests/unittests/arnold/arnold_test.py @@ -1,4 +1,5 @@ """Basic tests for nav.arnold""" + import unittest from nav.arnold import find_input_type diff --git a/tests/unittests/business/business_test.py b/tests/unittests/business/business_test.py index 6efcc1134a..0ca94bd66f 100644 --- a/tests/unittests/business/business_test.py +++ b/tests/unittests/business/business_test.py @@ -1,4 +1,5 @@ """Basic tests for nav.arnold""" + # pylint: disable=C0111 import unittest from datetime import datetime, timedelta diff --git a/tests/unittests/eventengine/severity_test.py b/tests/unittests/eventengine/severity_test.py index 749d014bcc..0fe0d2071c 100644 --- a/tests/unittests/eventengine/severity_test.py +++ b/tests/unittests/eventengine/severity_test.py @@ -161,7 +161,7 @@ def test_should_reset_severity_according_to_rule( def test_should_evaluate_to_single_rule_on_empty_rulesets( self, empty_ruleset_source ): - """Tests that an empty ruleset is parsed into a single default-value rule """ + """Tests that an empty ruleset is parsed into a single default-value rule""" rules = SeverityRules.load(empty_ruleset_source) assert len(rules) == 1 diff --git a/tests/unittests/ipdevpoll/plugins_paloaltoarp_test.py b/tests/unittests/ipdevpoll/plugins_paloaltoarp_test.py index f906a943dc..c22aa1bbf2 100644 --- a/tests/unittests/ipdevpoll/plugins_paloaltoarp_test.py +++ b/tests/unittests/ipdevpoll/plugins_paloaltoarp_test.py @@ -88,9 +88,10 @@ def test_do_request(): mock_agent = Mock(spec=Agent) mock_agent.request.return_value = succeed(mock_response) - with patch( - 'nav.ipdevpoll.plugins.paloaltoarp.Agent', return_value=mock_agent - ), patch('twisted.web.client.readBody', return_value="test content"): + with ( + patch('nav.ipdevpoll.plugins.paloaltoarp.Agent', return_value=mock_agent), + patch('twisted.web.client.readBody', return_value="test content"), + ): mock_address = "paloalto.example.org" mock_key = "secret" diff --git a/tests/unittests/ipdevpoll/timestamps_test.py b/tests/unittests/ipdevpoll/timestamps_test.py index 281c9719fd..df8aa5a5b0 100644 --- a/tests/unittests/ipdevpoll/timestamps_test.py +++ b/tests/unittests/ipdevpoll/timestamps_test.py @@ -1,4 +1,5 @@ """Tests for ipdevpoll's TimestampChecker utility class""" + from mock import Mock, patch import pytest diff --git a/tests/unittests/logs_test.py b/tests/unittests/logs_test.py index 016954093f..3fc25c17ff 100644 --- a/tests/unittests/logs_test.py +++ b/tests/unittests/logs_test.py @@ -1,4 +1,5 @@ """Tests for nav.logs module""" + import os from unittest import mock diff --git a/tests/unittests/metrics/thresholds_test.py b/tests/unittests/metrics/thresholds_test.py index ff168c5c96..b0b3f2b155 100644 --- a/tests/unittests/metrics/thresholds_test.py +++ b/tests/unittests/metrics/thresholds_test.py @@ -1,4 +1,5 @@ """Unit tests for nav.metrics.thresholds""" + from nav.metrics.thresholds import ThresholdEvaluator diff --git a/tests/unittests/portadmin/portadmin_util_test.py b/tests/unittests/portadmin/portadmin_util_test.py index 2f3f5a4157..b17fefdcc1 100644 --- a/tests/unittests/portadmin/portadmin_util_test.py +++ b/tests/unittests/portadmin/portadmin_util_test.py @@ -1,4 +1,5 @@ """Testing""" + import unittest from mock import Mock from nav.web.portadmin.utils import ( diff --git a/tests/unittests/report/util_test.py b/tests/unittests/report/util_test.py index 2ca2c5d581..2063ac16cd 100644 --- a/tests/unittests/report/util_test.py +++ b/tests/unittests/report/util_test.py @@ -1,4 +1,5 @@ """Tests for report view utility functions""" + from nav.web.report.views import find_page_range diff --git a/tools/eventgenerators/moduleevent.py b/tools/eventgenerators/moduleevent.py index 84ed2e03a7..5f0571b241 100755 --- a/tools/eventgenerators/moduleevent.py +++ b/tools/eventgenerators/moduleevent.py @@ -24,7 +24,7 @@ def handler(cursor, boxlist, state): - for (deviceid, netboxid, subid) in boxlist: + for deviceid, netboxid, subid in boxlist: sql = """INSERT INTO eventq (source, target, deviceid, netboxid, subid, eventtypeid, state,severity) @@ -65,7 +65,7 @@ def main(): box, module = spec.split(":") cursor.execute(sql, (box, module)) - for (deviceid, netboxid, moduleid, sysname, modulename) in cursor.fetchall(): + for deviceid, netboxid, moduleid, sysname, modulename in cursor.fetchall(): if not deviceid in device_dupes: netboxes.append((deviceid, netboxid, moduleid)) sysnames.append((sysname, modulename)) From 4ac49e44c48bb6e274e33fced4dc1cf0ae6ba08a Mon Sep 17 00:00:00 2001 From: Johanna England Date: Thu, 22 Aug 2024 16:25:15 +0200 Subject: [PATCH 3/6] Add black re-run to git blame ignore revs --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 1103ada44f..83a0ee9bf3 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -8,3 +8,5 @@ e6634e512c8ecf283c85a701366620e724806ab7 e0496b75789656f5abca770e928988fd09bf7803 # Re-ran black due to new version 24.4.2 18245ec1a042aac23c4c85637d3454e8d1b5a1bb +# Re-ran black due to new version 24.8.0 @johannaengland 22/08/2024 +90d33264b4d2aa106395f247e23830b0901bad3a From 85dde0388eb3ef26daa32386c1a7f5d41dcf1c3c Mon Sep 17 00:00:00 2001 From: Johanna England Date: Tue, 27 Aug 2024 13:06:58 +0200 Subject: [PATCH 4/6] Add excluded files to tox config --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 37c676541c..ce533d0e6b 100644 --- a/tox.ini +++ b/tox.ini @@ -141,3 +141,7 @@ statistics = True count = True # Only check for critical problems that will prevent testing/running select = E9,F63,F7,F82 +exclude= + python/nav/enterprise/ids.py, # Autogenerated list of constants + python/nav/smidumps/.*.py, # Autogenerated MIB dumps + python/nav/etc/geomap/config.py # Not actually a Python file From 4c83146e5628f0ee303f76c3447e75f6c35e6fd1 Mon Sep 17 00:00:00 2001 From: Johanna England Date: Tue, 27 Aug 2024 14:39:44 +0200 Subject: [PATCH 5/6] Add template to ignores files by flake8 --- tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index ce533d0e6b..326c2e755c 100644 --- a/tox.ini +++ b/tox.ini @@ -142,6 +142,7 @@ count = True # Only check for critical problems that will prevent testing/running select = E9,F63,F7,F82 exclude= - python/nav/enterprise/ids.py, # Autogenerated list of constants - python/nav/smidumps/.*.py, # Autogenerated MIB dumps - python/nav/etc/geomap/config.py # Not actually a Python file + python/nav/enterprise/ids.py, # Autogenerated list of constants + python/nav/smidumps/.*.py, # Autogenerated MIB dumps + python/nav/etc/geomap/config.py # Not actually a Python file + python/nav/snmptrapd/handlers/handlertemplate.py # Not in production, just a template From 371fa361107a40740592bfeb01f90c75d8cd7099 Mon Sep 17 00:00:00 2001 From: Johanna England Date: Tue, 27 Aug 2024 14:39:23 +0200 Subject: [PATCH 6/6] Add noqa to lines triggering undefined name --- python/nav/metrics/carbon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/nav/metrics/carbon.py b/python/nav/metrics/carbon.py index 9f67b3b6d3..b9480c973d 100644 --- a/python/nav/metrics/carbon.py +++ b/python/nav/metrics/carbon.py @@ -102,10 +102,10 @@ def _reset_warning_registry(): except NameError: pass else: - for key in __warningregistry__.keys(): + for key in __warningregistry__.keys(): # noqa _, cls, _ = key if cls is CarbonWarning: - del __warningregistry__[key] + del __warningregistry__[key] # noqa def send_metrics(metric_tuples):