From 5a9f36c76b5e313ee2f97b97aec5a718646e2e58 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Apr 2024 22:08:29 -0400 Subject: [PATCH 01/15] security update: gunicorn, magpie, docker base --- CHANGES.rst | 11 +++++++++-- docker/Dockerfile-base | 2 +- requirements.txt | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a0b46511..3650dc41 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,18 +7,25 @@ Changes `Unreleased `_ (latest) ------------------------------------------------------------------------------------ -* Nothing yet. +Features / Changes +~~~~~~~~~~~~~~~~~~~~~ +* Pin ``gunicorn>=22`` to address CVE-2024-1135. +* Pin Docker base to ``python:3.11-alpine3.19`` for various security fixes. +* Update ``magpie==4.0.0`` for corresponding fixes + (see `Changes: magpie @ 4.0.0 `_). `2.3.0 `_ (2023-11-30) ------------------------------------------------------------------------------------ +Features / Changes +~~~~~~~~~~~~~~~~~~~~~ * Add optional key ``field`` and ``regex`` to be used in the ``sync_permissions`` section found in the config. This allows to sync permissions using a field other than ``resource_full_name`` when creating the ``name:type`` from the segment ``ex.: /field1::type1/field2::type2``. Adds support to use ``resource_display_name``. * The ``regex`` is used to extract the desired information from the ``nametype_path``. It should be used to do an exact match. This new search overrides the default way of matching each segment with the ``nametype_path``. In the case where a ``regex`` is found in the target segment, the data will be formed using the same ``resource_type`` - for every match in the same segment. Similary, as using ``- name: "**"`` in the config to match multiple segment, + for every match in the same segment. Similarly, as using ``- name: "**"`` in the config to match multiple segment, it is possible to use a ``regex`` to match multiple resources in the same segment with ``regex: '(?<=:).*\/?(?=\/)'`` `2.2.0 `_ (2023-11-16) diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base index e2b126ab..7907c4e7 100644 --- a/docker/Dockerfile-base +++ b/docker/Dockerfile-base @@ -1,4 +1,4 @@ -FROM python:3.10-alpine +FROM python:3.11-alpine3.19 LABEL description.short="Cowbird base" LABEL description.long="Cowbird middleware to manage interactions between various birds of the bird-house stack." LABEL Maintainer="Ouranos, CRIM" diff --git a/requirements.txt b/requirements.txt index 0ba969f0..756617e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,9 +5,8 @@ cornice>=5 cornice_swagger>=0.7.0 dicttoxml>=1.7.16 # flag --paste breaks for >20 unless using pserve -gunicorn<20; python_version < "3.5" # pyup: ignore gunicorn>=20 -magpie @ git+https://github.com/Ouranosinc/Magpie@3.34.0 +magpie @ git+https://github.com/Ouranosinc/Magpie@4.0.0 paste pastedeploy pymongo[srv]>=4.4,<5 # required to work with pinned celery From 5c30a422f29bcc0327bb00d0fff0f7b802960b9c Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Apr 2024 22:43:54 -0400 Subject: [PATCH 02/15] bump sphinx attempt fix readthedocs build --- requirements-doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index 8bcbd333..e29ec2b5 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -8,7 +8,7 @@ astroid>=2.11.0,<3 jinja2<3.1 # Avoid jinja2 3.1 version, which breaks with sphinx 3.X (https://github.com/sphinx-doc/sphinx/issues/10291) sphinx-autoapi>=2.1.0 sphinx-paramlinks>=0.4.1 -sphinx==7.1 +sphinx>=7.1,<8 sphinxcontrib-redoc>=1.6.0 sphinx_autodoc_typehints[type-comment]>=1.19.3,<1.23.1 sphinx_rtd_theme From 83fabe1b6783eb5ec22e60331f23028917400461 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Apr 2024 22:53:50 -0400 Subject: [PATCH 03/15] update docs reqs --- requirements-doc.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index e29ec2b5..d7235530 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -5,10 +5,10 @@ -r requirements.txt # these are doc-only requirements astroid>=2.11.0,<3 -jinja2<3.1 # Avoid jinja2 3.1 version, which breaks with sphinx 3.X (https://github.com/sphinx-doc/sphinx/issues/10291) +jinja2==3.1.3 sphinx-autoapi>=2.1.0 sphinx-paramlinks>=0.4.1 sphinx>=7.1,<8 sphinxcontrib-redoc>=1.6.0 -sphinx_autodoc_typehints[type-comment]>=1.19.3,<1.23.1 +sphinx_autodoc_typehints[type-comment]>=1.25 sphinx_rtd_theme From 000832848ed0e810da74087c11f2277d966f9330 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Apr 2024 23:44:27 -0400 Subject: [PATCH 04/15] adjust npm based linters --- .github/ISSUE_TEMPLATE/bug-report.md | 29 +++++---- .github/ISSUE_TEMPLATE/feature-request.md | 28 ++++++--- .gitignore | 2 +- .remarkignore | 13 ++++ .stylelintrc.json | 13 ---- Makefile | 68 +++++++++++++++----- docker/hooks/README.md | 2 +- package.json | 77 +++++++++++++++++++++++ 8 files changed, 182 insertions(+), 50 deletions(-) create mode 100644 .remarkignore delete mode 100644 .stylelintrc.json create mode 100644 package.json diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 9568ec85..e7ea7c1c 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,30 +1,35 @@ --- name: Bug Report about: Create a report to help us improve -title: \[BUG\] +title: \[BUG] labels: bug assignees: fmigneault - --- -**Describe the bug** +## Describe the bug + + + +## To Reproduce -**To Reproduce** Steps to reproduce the behavior: + 1. Deploy process with payload '...' 2. Execute using payload '....' 3. Result '....' 4. Error message '...' -**Expected behavior** +## Expected behavior + + -**Screenshots** -If applicable, add screenshots to help explain your problem. +## Environment -**Desktop (please complete the following information):** - - OS: \[e.g. Linux|Windows\] (if running locally) - - Browser \[e.g. chrome, safari\] (if running as a service) - - Instance: URL - - Version \["1.2.3", see `/version` endpoint\] +- OS: \[e.g. Linux|Windows] (if running locally) +- Browser \[e.g. chrome, safari] (if running as a service) +- Instance: URL +- Version \["1.2.3", see `/version` endpoint] diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 179274d9..bd0bef45 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,21 +1,31 @@ --- name: Feature Request about: Suggest an idea for this project -title: \[Feature\] +title: \[Feature] labels: feature assignees: fmigneault - --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. -Ex. I would like to be able to authenticate using \[...\] +## Describe the request + + + +## Expected behavior -**Describe the solution you'd like** -A clear and concise description of what you want to happen. + -**Describe alternatives you've considered** +## Alternatives considered + + + +## Additional context -**Additional context** + diff --git a/.gitignore b/.gitignore index e1c3cab4..9d2ae05d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ share ## Node node_modules -package.json +!package.json package-lock.json ## Makefile diff --git a/.remarkignore b/.remarkignore new file mode 100644 index 00000000..1b6d93e1 --- /dev/null +++ b/.remarkignore @@ -0,0 +1,13 @@ +# To save time scanning +.idea/ +.vscode/ +*.egg-info/ +downloads/ +env/ + +# actual items to ignore +.pytest_cache/ +node_modules/ +docs/_build/ +docs/build/ +reports/ diff --git a/.stylelintrc.json b/.stylelintrc.json deleted file mode 100644 index cb464aab..00000000 --- a/.stylelintrc.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "stylelint-config-standard", - "ignoreFiles": ["docs/_build/**"], - "rules": { - "block-no-empty": null, - "color-no-invalid-hex": true, - "color-hex-case": "upper", - "color-hex-length": "long", - "indentation": [4], - "no-descending-specificity": null, - "property-no-vendor-prefix": null - } -} diff --git a/Makefile b/Makefile index 31c3d8f0..6cf97dee 100644 --- a/Makefile +++ b/Makefile @@ -341,18 +341,24 @@ install-dev: conda-env install-xargs ## install package requirements for develop # install locally to ensure they can be found by config extending them .PHONY: install-npm -install-npm: ## install npm package manager if it cannot be found +install-npm: ## install npm package manager and dependencies if they cannot be found @[ -f "$(shell which npm)" ] || ( \ echo "Binary package manager npm not found. Attempting to install it."; \ apt-get install npm \ ) - @[ `npm ls 2>/dev/null | grep stylelint-config-standard | wc -l` = 1 ] || ( \ - echo "Install required libraries for style checks." && \ + +.PHONY: install-npm-stylelint +install-npm-stylelint: install-npm ## install stylelint dependency for 'check-css' target using npm + @[ `npm ls 2>/dev/null | grep stylelint-config-standard | grep -v UNMET | wc -l` = 1 ] || ( \ + echo "Install required dependencies for CSS checks." && \ npm install --save-dev \ - stylelint \ - stylelint-scss \ - stylelint-config-standard \ - stylelint-csstree-validator \ + ) + +.PHONY: install-npm-remarklint +install-npm-remarklint: install-npm ## install remark-lint dependency for 'check-md' target using npm + @[ `npm ls 2>/dev/null | grep remark-lint | grep -v UNMET | wc -l` = 1 ] || ( \ + echo "Install required dependencies for Markdown checks." && \ + npm install --save-dev ) ## --- Launchers targets --- ## @@ -512,7 +518,7 @@ mkdir-reports: # autogen check variants with pre-install of dependencies using the '-only' target references CHECKS_EXCLUDE ?= CHECKS_PYTHON := pep8 lint security doc8 docf links imports types -CHECKS_NPM := css +CHECKS_NPM := css md CHECKS_PYTHON := $(filter-out $(CHECKS_EXCLUDE),$(CHECKS_PYTHON)) CHECKS_NPM := $(filter-out $(CHECKS_EXCLUDE),$(CHECKS_NPM)) CHECKS := $(CHECKS_PYTHON) $(CHECKS_NPM) @@ -606,15 +612,34 @@ check-types-only: mkdir-reports ## run typing validation .PHONY: check-css-only check-css-only: mkdir-reports @echo "Running CSS style checks..." - @npx stylelint \ - --config "$(APP_ROOT)/.stylelintrc.json" \ + @npx --no-install stylelint \ + --config "$(APP_ROOT)/package.json" \ --output-file "$(REPORTS_DIR)/fixed-css.txt" \ "$(APP_ROOT)/**/*.css" +.PHONY: check-css +check-css: install-npm-stylelint check-css-only ## check CSS linting after dependency installation + +# must pass 2 search paths because '/.' are somehow not correctly detected with only the top-level +.PHONY: check-md-only +check-md-only: mkdir-reports ## check Markdown linting + @echo "Running Markdown style checks..." + @npx --no-install remark \ + --inspect --frail \ + --silently-ignore \ + --stdout --color \ + --rc-path "$(APP_ROOT)/package.json" \ + --ignore-path "$(APP_ROOT)/.remarkignore" \ + "$(APP_ROOT)" "$(APP_ROOT)/.*/" \ + > "$(REPORTS_DIR)/check-md.txt" + +.PHONY: check-md +check-md: install-npm-remarklint check-md-only ## check Markdown linting after dependency installation + # autogen fix variants with pre-install of dependencies using the '-only' target references FIXES_EXCLUDE ?= FIXES_PYTHON := imports lint docf fstring -FIXES_NPM := css +FIXES_NPM := css md FIXES_PYTHON := $(filter-out $(FIXES_EXCLUDE),$(FIXES_PYTHON)) FIXES_NPM := $(filter-out $(FIXES_EXCLUDE),$(FIXES_NPM)) FIXES := $(FIXES_PYTHON) $(FIXES_NPM) @@ -672,17 +697,32 @@ fix-fstring-only: mkdir-reports ## fix code string formats substitutions to f-s 1> >(tee "$(REPORTS_DIR)/fixed-fstring.txt")' .PHONY: fix-css -fix-css: install-npm fix-css-only +fix-css: install-npm-stylelint fix-css-only .PHONY: fix-css-only fix-css-only: mkdir-reports ## fix CSS styles problems automatically @echo "Fixing CSS style problems..." - @npx stylelint \ + @npx --no-install stylelint \ --fix \ - --config "$(APP_ROOT)/.stylelintrc.json" \ + --config "$(APP_ROOT)/package.json" \ --output-file "$(REPORTS_DIR)/fixed-css.txt" \ "$(APP_ROOT)/**/*.css" +# must pass 2 search paths because '/.' are somehow not correctly detected with only the top-level +.PHONY: fix-md-only +fix-md-only: mkdir-reports ## fix Markdown linting problems automatically + @echo "Running Markdown style checks..." + @npx --no-install remark \ + --output --frail \ + --silently-ignore \ + --rc-path "$(APP_ROOT)/package.json" \ + --ignore-path "$(APP_ROOT)/.remarkignore" \ + "$(APP_ROOT)" "$(APP_ROOT)/.*/" \ + 2>&1 | tee "$(REPORTS_DIR)/fixed-md.txt" + +.PHONY: fix-md +fix-md: install-npm-remarklint fix-md-only ## fix Markdown linting problems after dependency installation + ## --- Test targets --- ## .PHONY: test diff --git a/docker/hooks/README.md b/docker/hooks/README.md index 75d7507c..89bbba27 100644 --- a/docker/hooks/README.md +++ b/docker/hooks/README.md @@ -1,4 +1,4 @@ Files placed in this directory are for additional/override DockerHub auto-build hook procedures. -see: https://docs.docker.com/docker-hub/builds/advanced/ +see: [https://docs.docker.com/docker-hub/builds/advanced/](https://docs.docker.com/docker-hub/builds/advanced/) Hook directory must be placed as sibling to the referenced Dockerfile location in the automated build configuration. diff --git a/package.json b/package.json new file mode 100644 index 00000000..f805786a --- /dev/null +++ b/package.json @@ -0,0 +1,77 @@ +{ + "devDependencies": { + "remark-cli": "^12.0.0", + "remark-frontmatter": "^5.0.0", + "remark-gfm": "^4.0.0", + "remark-lint": "^9.1.2", + "remark-lint-checkbox-content-indent": "^4.1.2", + "remark-lint-maximum-line-length": "^3.1.3", + "remark-preset-lint-markdown-style-guide": "^5.1.3", + "remark-preset-lint-recommended": "^6.1.3", + "stylelint": "^15.11.0", + "stylelint-config-standard": "^34.0.0", + "stylelint-scss": "^5.3.2", + "stylelint-csstree-validator": "^3.0.0" + }, + "remarkConfig": { + "settings": { + "bullet": "-", + "fence": "`", + "fences": "true", + "listItemIndent": "mixed", + "incrementListMarker": "true", + "resourceLink": "true", + "rule": "-" + }, + "plugins": [ + "remark-gfm", + "remark-frontmatter", + "remark-preset-lint-markdown-style-guide", + "remark-preset-lint-recommended", + "remark-lint-list-item-content-indent", + "remark-lint-checkbox-content-indent", + [ + "lint-fenced-code-marker", + "`" + ], + [ + "lint-list-item-indent", + "mixed" + ], + [ + "lint-maximum-line-length", + 120 + ], + [ + "lint-ordered-list-marker-style", + "." + ], + [ + "lint-ordered-list-marker-value", + "ordered" + ], + [ + "lint-unordered-list-marker-style", + "consistent" + ] + ] + }, + "stylelint": { + "extends": "stylelint-config-standard", + "ignoreFiles": [ + "docs/_build/**", + "docs/build/**" + ], + "rules": { + "block-no-empty": null, + "color-no-invalid-hex": true, + "color-hex-case": "upper", + "color-hex-length": "long", + "indentation": [ + 4 + ], + "property-no-vendor-prefix": null, + "no-descending-specificity": null + } + } +} From 63646986f928804df16a21e4638ed37a1006221d Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Apr 2024 23:56:01 -0400 Subject: [PATCH 05/15] adjust sphinx-autoapi --- requirements-doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index d7235530..3862e86a 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -6,7 +6,7 @@ # these are doc-only requirements astroid>=2.11.0,<3 jinja2==3.1.3 -sphinx-autoapi>=2.1.0 +sphinx-autoapi>=2.1.0,<3 sphinx-paramlinks>=0.4.1 sphinx>=7.1,<8 sphinxcontrib-redoc>=1.6.0 From 7db05994484a2cbe7286b147a536f2eb5e083948 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Sat, 27 Apr 2024 00:08:31 -0400 Subject: [PATCH 06/15] undo pin sphinx-autoapi --- cowbird/permissions_synchronizer.py | 11 +++++++---- requirements-doc.txt | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cowbird/permissions_synchronizer.py b/cowbird/permissions_synchronizer.py index 6e428b90..60f2e58e 100644 --- a/cowbird/permissions_synchronizer.py +++ b/cowbird/permissions_synchronizer.py @@ -525,10 +525,13 @@ def _get_permission_data(self, .. code-block:: json - { : { - "res_path": [], - "permissions": { : [user, grp], ...}}, - ... + { + "": { + "res_path": [""], + "permissions": { + "": [{"user": "...", "group": "..."}] + } + } } """ permission_data: PermissionData = {} diff --git a/requirements-doc.txt b/requirements-doc.txt index 3862e86a..d7235530 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -6,7 +6,7 @@ # these are doc-only requirements astroid>=2.11.0,<3 jinja2==3.1.3 -sphinx-autoapi>=2.1.0,<3 +sphinx-autoapi>=2.1.0 sphinx-paramlinks>=0.4.1 sphinx>=7.1,<8 sphinxcontrib-redoc>=1.6.0 From 76ea3d54b459fdf8ed320faf91b6fd4bcb2f15b4 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 16:14:30 -0400 Subject: [PATCH 07/15] ignore broken link --- docs/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3838f124..24847aaf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -150,12 +150,13 @@ def doc_redirect_include(file_path): "https://github.com/Ouranosinc/cowbird/*", # limit only Cowbird so others are still checked # ignore private links "https://github.com/Ouranosinc/PAVICS/*", - "CHANGES.rst", + ".*CHANGES.rst.*", "AUTHORS.rst", r"docs/\w+.rst", ] linkcheck_anchors_ignore = [ - r".*issuecomment.*" # github issue comment anchors not resolved + r".*issuecomment.*", # GitHub issue comment anchors not resolved + r".*CHANGES.rst.*", # GitHub anchor to specific changelog version ] linkcheck_anchors_ignore_for_url = [ From a8689d42b6070fc72a51a7d7d1ab58c2baa173e6 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 20:54:49 -0400 Subject: [PATCH 08/15] ignore typing issues + update watchdog package --- cowbird/monitoring/monitor.py | 8 ++++---- cowbird/typedefs.py | 2 +- requirements.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cowbird/monitoring/monitor.py b/cowbird/monitoring/monitor.py index 15e56551..5b941662 100644 --- a/cowbird/monitoring/monitor.py +++ b/cowbird/monitoring/monitor.py @@ -170,7 +170,7 @@ def stop(self) -> None: self.__event_observer.join() self.__event_observer = None - def on_moved(self, event: Union[DirMovedEvent, FileMovedEvent]) -> None: + def on_moved(self, event: Union[DirMovedEvent, FileMovedEvent]) -> None: # type: ignore[override] """ Called when a file or a directory is moved or renamed. @@ -186,7 +186,7 @@ def on_moved(self, event: Union[DirMovedEvent, FileMovedEvent]) -> None: os.path.dirname(self.__src_path): self.__callback.on_created(event.dest_path) - def on_created(self, event: Union[DirCreatedEvent, FileCreatedEvent]) -> None: + def on_created(self, event: Union[DirCreatedEvent, FileCreatedEvent]) -> None: # type: ignore[override] """ Called when a file or directory is created. @@ -194,7 +194,7 @@ def on_created(self, event: Union[DirCreatedEvent, FileCreatedEvent]) -> None: """ self.__callback.on_created(event.src_path) - def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None: + def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None: # type: ignore[override] """ Called when a file or directory is deleted. @@ -202,7 +202,7 @@ def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None: """ self.__callback.on_deleted(event.src_path) - def on_modified(self, event: Union[DirModifiedEvent, FileModifiedEvent]) -> None: + def on_modified(self, event: Union[DirModifiedEvent, FileModifiedEvent]) -> None: # type: ignore[override] """ Called when a file or directory is modified. diff --git a/cowbird/typedefs.py b/cowbird/typedefs.py index 1699caae..de13177b 100644 --- a/cowbird/typedefs.py +++ b/cowbird/typedefs.py @@ -58,7 +58,7 @@ ParamsType = Dict[str, Any] CookiesType = Union[Dict[str, str], List[Tuple[str, str]]] HeadersType = Union[Dict[str, str], List[Tuple[str, str]]] -AnyHeadersType = Union[HeadersType, ResponseHeaders, EnvironHeaders, CaseInsensitiveDict] +AnyHeadersType = Union[HeadersType, ResponseHeaders, EnvironHeaders, CaseInsensitiveDict] # type: ignore[type-arg] AnyCookiesType = Union[CookiesType, RequestsCookieJar] AnyResponseType = Union[WebobResponse, PyramidResponse, RequestsResponse, HTTPException, TestResponseType] diff --git a/requirements.txt b/requirements.txt index 756617e4..96b9c34d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,5 @@ schema simplejson threddsclient==0.4.4 typing_extensions -watchdog +watchdog>=4 webob From 90a9c8d04eb862f6d8bb4fb232ec63d63b2da30b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 20:54:49 -0400 Subject: [PATCH 09/15] ignore typing issues + update watchdog package --- CHANGES.rst | 1 + cowbird/monitoring/monitor.py | 8 ++++---- cowbird/typedefs.py | 2 +- requirements.txt | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3650dc41..b6e8f312 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,7 @@ Features / Changes * Pin Docker base to ``python:3.11-alpine3.19`` for various security fixes. * Update ``magpie==4.0.0`` for corresponding fixes (see `Changes: magpie @ 4.0.0 `_). +* Pin ``watchdog>=4`` for latest typing additions. `2.3.0 `_ (2023-11-30) ------------------------------------------------------------------------------------ diff --git a/cowbird/monitoring/monitor.py b/cowbird/monitoring/monitor.py index 15e56551..5b941662 100644 --- a/cowbird/monitoring/monitor.py +++ b/cowbird/monitoring/monitor.py @@ -170,7 +170,7 @@ def stop(self) -> None: self.__event_observer.join() self.__event_observer = None - def on_moved(self, event: Union[DirMovedEvent, FileMovedEvent]) -> None: + def on_moved(self, event: Union[DirMovedEvent, FileMovedEvent]) -> None: # type: ignore[override] """ Called when a file or a directory is moved or renamed. @@ -186,7 +186,7 @@ def on_moved(self, event: Union[DirMovedEvent, FileMovedEvent]) -> None: os.path.dirname(self.__src_path): self.__callback.on_created(event.dest_path) - def on_created(self, event: Union[DirCreatedEvent, FileCreatedEvent]) -> None: + def on_created(self, event: Union[DirCreatedEvent, FileCreatedEvent]) -> None: # type: ignore[override] """ Called when a file or directory is created. @@ -194,7 +194,7 @@ def on_created(self, event: Union[DirCreatedEvent, FileCreatedEvent]) -> None: """ self.__callback.on_created(event.src_path) - def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None: + def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None: # type: ignore[override] """ Called when a file or directory is deleted. @@ -202,7 +202,7 @@ def on_deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None: """ self.__callback.on_deleted(event.src_path) - def on_modified(self, event: Union[DirModifiedEvent, FileModifiedEvent]) -> None: + def on_modified(self, event: Union[DirModifiedEvent, FileModifiedEvent]) -> None: # type: ignore[override] """ Called when a file or directory is modified. diff --git a/cowbird/typedefs.py b/cowbird/typedefs.py index 1699caae..de13177b 100644 --- a/cowbird/typedefs.py +++ b/cowbird/typedefs.py @@ -58,7 +58,7 @@ ParamsType = Dict[str, Any] CookiesType = Union[Dict[str, str], List[Tuple[str, str]]] HeadersType = Union[Dict[str, str], List[Tuple[str, str]]] -AnyHeadersType = Union[HeadersType, ResponseHeaders, EnvironHeaders, CaseInsensitiveDict] +AnyHeadersType = Union[HeadersType, ResponseHeaders, EnvironHeaders, CaseInsensitiveDict] # type: ignore[type-arg] AnyCookiesType = Union[CookiesType, RequestsCookieJar] AnyResponseType = Union[WebobResponse, PyramidResponse, RequestsResponse, HTTPException, TestResponseType] diff --git a/requirements.txt b/requirements.txt index 756617e4..96b9c34d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,5 @@ schema simplejson threddsclient==0.4.4 typing_extensions -watchdog +watchdog>=4 webob From 5c293d0c1c160e9d631efb01b283ec05646619ce Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 21:05:48 -0400 Subject: [PATCH 10/15] update jinja for docs --- requirements-doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index d7235530..f8535f34 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -5,7 +5,7 @@ -r requirements.txt # these are doc-only requirements astroid>=2.11.0,<3 -jinja2==3.1.3 +jinja2>=3.1.4,<3.2 sphinx-autoapi>=2.1.0 sphinx-paramlinks>=0.4.1 sphinx>=7.1,<8 From e243938229cf46862a0ae51ed15114860b7f8555 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 21:07:47 -0400 Subject: [PATCH 11/15] ignore pyup for jinja in docs --- requirements-doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index f8535f34..f7b6f1fc 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -5,7 +5,7 @@ -r requirements.txt # these are doc-only requirements astroid>=2.11.0,<3 -jinja2>=3.1.4,<3.2 +jinja2>=3.1.4,<3.2 # pyup: ignore # all versions flagged, no fix possible! sphinx-autoapi>=2.1.0 sphinx-paramlinks>=0.4.1 sphinx>=7.1,<8 From ba41451b2bed4c40702999ba556d7128444f4890 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 21:25:25 -0400 Subject: [PATCH 12/15] combine CI tests + add missing npm deps for CI tests --- .github/workflows/tests.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 433c4cfa..b74c390d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,7 +29,6 @@ jobs: env: # override make command to install directly in active python CONDA_COMMAND: "" - CHECKS_EXCLUDE: types strategy: matrix: os: [ubuntu-latest] @@ -43,10 +42,6 @@ jobs: python-version: "3.10" allow-failure: false test-case: check-only - - os: ubuntu-latest - python-version: "3.10" - allow-failure: true - test-case: check-types-only # docker smoke tests - os: ubuntu-latest python-version: "3.10" @@ -89,7 +84,7 @@ jobs: run: make install-pkg install-dev - if: ${{ matrix.test-case == 'check-only' }} name: Install Extra Test Dependencies - run: make install-npm + run: make install-npm install-npm-stylelint install-npm-remarklint - name: Display Packages run: pip freeze - name: Display Version From dd85cfe1cdcf24e3afc6613f2918d437c6e8468e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 21:30:14 -0400 Subject: [PATCH 13/15] update dev target definitions for simplified invocation by CI --- .github/workflows/tests.yml | 2 +- Makefile | 36 +++++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b74c390d..443468db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -84,7 +84,7 @@ jobs: run: make install-pkg install-dev - if: ${{ matrix.test-case == 'check-only' }} name: Install Extra Test Dependencies - run: make install-npm install-npm-stylelint install-npm-remarklint + run: make install-dev - name: Display Packages run: pip freeze - name: Display Version diff --git a/Makefile b/Makefile index 6cf97dee..4c4f5652 100644 --- a/Makefile +++ b/Makefile @@ -334,8 +334,8 @@ install-docs: conda-env install-xargs ## install package requirements for docum @bash -c '$(CONDA_CMD) pip install $(PIP_XARGS) -r "$(APP_ROOT)/requirements-doc.txt"' @echo "Successfully installed docs requirements." -.PHONY: install-dev -install-dev: conda-env install-xargs ## install package requirements for development and testing +.PHONY: install-dev-python +install-dev-python: conda-env install-xargs ## install all Python package requirements for development and testing @bash -c '$(CONDA_CMD) pip install $(PIP_XARGS) -r "$(APP_ROOT)/requirements-dev.txt"' @echo "Successfully installed dev requirements." @@ -361,6 +361,12 @@ install-npm-remarklint: install-npm ## install remark-lint dependency for 'chec npm install --save-dev ) +.PHONY: install-dev-npm +install-dev-npm: install-npm install-npm-remarklint install-npm-remarklint ## install all npm development dependencies + +.PHONY: install-dev +install-dev: install-dev-python install-dev-npm ## install all development dependencies + ## --- Launchers targets --- ## .PHONY: cron @@ -525,10 +531,10 @@ CHECKS := $(CHECKS_PYTHON) $(CHECKS_NPM) CHECKS := $(addprefix check-, $(CHECKS)) CHECKS_PYTHON := $(addprefix check-, $(CHECKS_PYTHON)) -$(CHECKS_PYTHON): check-%: install-dev check-%-only +$(CHECKS_PYTHON): check-%: install-dev-python check-%-only CHECKS_NPM := $(addprefix check-, $(CHECKS_NPM)) -$(CHECKS_NPM): check-%: install-npm check-%-only +$(CHECKS_NPM): check-%: install-dev-npm check-%-only .PHONY: check check: check-all ## alias for 'check-all' target @@ -646,10 +652,10 @@ FIXES := $(FIXES_PYTHON) $(FIXES_NPM) FIXES := $(addprefix fix-, $(FIXES)) FIXES_PYTHON := $(addprefix fix-, $(FIXES_PYTHON)) -$(FIXES_PYTHON): fix-%: install-dev fix-%-only +$(FIXES_PYTHON): fix-%: install-dev-python fix-%-only FIXES_NPM := $(addprefix fix-, $(FIXES_NPM)) -$(FIXES_NPM): fix-%: install-npm fix-%-only +$(FIXES_NPM): fix-%: install-dev-npm fix-%-only .PHONY: fix fix: fix-all ## alias for 'fix-all' target @@ -729,10 +735,10 @@ fix-md: install-npm-remarklint fix-md-only ## fix Markdown linting problems afte test: test-all ## alias for 'test-all' target .PHONY: test-all -test-all: install-dev install test-only ## run all tests combinations +test-all: install-dev-python install test-only ## run all tests combinations .PHONY: test-all -test-all: install-dev install test-only ## run all tests combinations +test-all: install-dev-python install test-only ## run all tests combinations .PHONY: test-only test-only: ## run all tests, but without prior dependency check and installation @@ -740,28 +746,28 @@ test-only: ## run all tests, but without prior dependency check and installatio @bash -c '$(CONDA_CMD) pytest tests -vv --junitxml "$(APP_ROOT)/tests/results.xml"' .PHONY: test-api -test-api: install-dev install ## run only API tests with the environment Python +test-api: install-dev-python install ## run only API tests with the environment Python @echo "Running local tests..." @bash -c '$(CONDA_CMD) pytest tests -vv -m "api" --junitxml "$(APP_ROOT)/tests/results.xml"' .PHONY: test-cli -test-cli: install-dev install ## run only CLI tests with the environment Python +test-cli: install-dev-python install ## run only CLI tests with the environment Python @echo "Running local tests..." @bash -c '$(CONDA_CMD) pytest tests -vv -m "cli" --junitxml "$(APP_ROOT)/tests/results.xml"' .PHONY: test-geoserver -test-geoserver: install-dev install ## run Geoserver requests tests against a configured Geoserver instance. Most of these tests are "online" tests +test-geoserver: install-dev-python install ## run Geoserver requests tests against a configured Geoserver instance. Most of these tests are "online" tests @echo "Running local tests..." @bash -c '$(CONDA_CMD) pytest tests -vv -m "geoserver" --junitxml "$(APP_ROOT)/tests/results.xml"' .PHONY: test-magpie -test-magpie: install-dev install ## run Magpie requests tests against a configured Magpie instance. Most of these tests are "online" tests +test-magpie: install-dev-python install ## run Magpie requests tests against a configured Magpie instance. Most of these tests are "online" tests @echo "Running local tests..." @bash -c '$(CONDA_CMD) pytest tests -vv -m "magpie" --junitxml "$(APP_ROOT)/tests/results.xml"' .PHONY: test-custom -test-custom: install-dev install ## run custom marker tests using SPEC="" +test-custom: install-dev-python install ## run custom marker tests using SPEC="" @echo "Running custom tests..." @[ "${SPEC}" ] || ( echo ">> 'TESTS' is not set"; exit 1 ) @bash -c '$(CONDA_CMD) pytest tests -vv -m "${SPEC}" --junitxml "$(APP_ROOT)/tests/results.xml"' @@ -773,7 +779,7 @@ test-docker: docker-test ## alias for 'docker-test' target [WARNING: could bui COVERAGE_FILE := $(APP_ROOT)/.coverage COVERAGE_HTML_DIR := $(REPORTS_DIR)/coverage COVERAGE_HTML_IDX := $(COVERAGE_HTML_DIR)/index.html -$(COVERAGE_FILE): install-dev +$(COVERAGE_FILE): install-dev-python @echo "Running coverage analysis..." @bash -c '$(CONDA_CMD) coverage run --source "$(APP_ROOT)/$(APP_NAME)" \ `which pytest` tests -m "not remote" || true' @@ -783,7 +789,7 @@ $(COVERAGE_FILE): install-dev @-echo "Coverage report available: file://$(COVERAGE_HTML_IDX)" .PHONY: coverage -coverage: install-dev install $(COVERAGE_FILE) ## check code coverage and generate an analysis report +coverage: install-dev-python install $(COVERAGE_FILE) ## check code coverage and generate an analysis report .PHONY: coverage-show coverage-show: $(COVERAGE_HTML_IDX) ## display HTML webpage of generated coverage report (run coverage if missing) From dabf1ab4659f62247ef165566929b8e0d91dd431 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 21:31:21 -0400 Subject: [PATCH 14/15] fix missing backslack in install-npm-remarklint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4c4f5652..e8cac7ab 100644 --- a/Makefile +++ b/Makefile @@ -358,7 +358,7 @@ install-npm-stylelint: install-npm ## install stylelint dependency for 'check-cs install-npm-remarklint: install-npm ## install remark-lint dependency for 'check-md' target using npm @[ `npm ls 2>/dev/null | grep remark-lint | grep -v UNMET | wc -l` = 1 ] || ( \ echo "Install required dependencies for Markdown checks." && \ - npm install --save-dev + npm install --save-dev \ ) .PHONY: install-dev-npm From 6791ba007ec5645226c0adc538546047b6974a0b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 19 Jun 2024 21:42:15 -0400 Subject: [PATCH 15/15] add CODECOV_TOKEN to CI --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 443468db..7ccf41af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -101,9 +101,10 @@ jobs: run: make stop ${{ matrix.test-case }} continue-on-error: false - name: Upload coverage report - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 if: ${{ success() && matrix.test-case == 'coverage' }} with: + token: ${{ secrets.CODECOV_TOKEN }} files: ./reports/coverage.xml fail_ci_if_error: true verbose: true