From a3452b3c6aae23e473254023ca5cda0756d5400e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Thu, 3 Oct 2024 12:57:24 +0200 Subject: [PATCH 1/4] Define project and dependencies to pyproject.toml The primary motivation for this is to get the Github dependency graph to resolve properly so that dependabot security alerts can work in this repository. Apparently the system that resolves the dependency graph can be unreliable when dependencies are defined in setup.py[1], and using variables when specifying dependencies may also cause issues[2]. pyproject.toml / PEP621 seems like the new standard of configuring python projects and associated tooling, so I think this is a step in the right direction in any case. A minor difference is that the `[dev]` dependency target doesn't include the dependencies from the `[ci]` target. To work around that, it is necessary to use `[dev,ci]` to achieve the same thing as `[dev]` previously. requirements.txt is updated to reflect this, so `pip install -r requirements.txt` should still install all dependencies. [1]: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-the-dependency-graph#supported-package-ecosystems [2]: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/troubleshooting-the-dependency-graph#does-the-dependency-graph-detect-dependencies-specified-using-variables --- pyproject.toml | 85 ++++++++++++++++++++++++++++++++++++ pytest.ini | 4 -- requirements.txt | 2 +- setup.cfg | 10 ----- setup.py | 111 ----------------------------------------------- 5 files changed, 86 insertions(+), 126 deletions(-) delete mode 100644 pytest.ini delete mode 100755 setup.py diff --git a/pyproject.toml b/pyproject.toml index 55ec8d78..e498f8a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,87 @@ +[build-system] +requires = [ + "setuptools>=61.0", + "wheel", +] +build-backend = "setuptools.build_meta" + + +[tool.setuptools.packages.find] +exclude = ["tests"] + +[project] +name = "fiaas-deploy-daemon" +version = "1.0.dev" +authors = [ + {email = "fiaas@googlegroups.com"}, +] +description = "Deploy applications to Kubernetes" +readme = "README.md" +license = {file = "LICENSE"} +requires-python = ">= 3.12" +dependencies = [ + "ConfigArgParse == 0.14.0", + "prometheus_client == 0.7.1", + "PyYAML == 5.1.2", + "pyaml == 19.4.1", + "pinject == 0.14.1", + "decorator < 5.0.0", # 5.0.0 and later drops py2 support (transitive dep from pinject) + "six >= 1.12.0", + "dnspython == 1.16.0", + "k8s == 0.27.1", + "appdirs == 1.4.3", + "requests-toolbelt == 0.10.1", + "backoff == 1.8.0", + "py27hash == 1.1.0", + "Flask == 3.0.0", + "flask-talisman >= 1.1.0", + "jinja2 >= 3.0.1", + "markupsafe >= 2.1.3", + "itsdangerous >= 2.1.2", + "werkzeug >= 3.0.1", + "blinker >= 1.7.0", + "urllib3 == 1.26.17", + "requests == 2.31.0", + "ipaddress == 1.0.22", # Required by requests for resolving IP address in SSL cert +] + +[project.optional-dependencies] +# Tooling for development and running tests +dev = [ + "flake8-print == 3.1.4", + "flake8-comprehensions == 1.4.1", + "pep8-naming == 0.11.1", + "flake8 == 7.0.0", + "pytest-xdist == 3.6.1", + "pytest-sugar == 1.0.0", + "pytest-html == 4.1.1", + "pytest-cov == 5.0.0", + "pytest-helpers-namespace == 2021.12.29", + "pytest == 8.2.0", + "requests-file == 1.4.3", + "callee == 0.3.1", +] +# CI builds use tox to run tests; tox will install the rest of the dependencies in its own virtualenvs +ci = [ + "tox==4.14.2", + "virtualenv==20.26.0", + "black ~= 22.0", +] + +[project.scripts] +fiaas-deploy-daemon = "fiaas_deploy_daemon:main" +fiaas-deploy-daemon-bootstrap = "fiaas_deploy_daemon.bootstrap:main" + [tool.black] line-length = 120 + +[tool.coverage.html] +directory = "build/reports/coverage" + +[tool.coverage.xml] +output = "build/reports/coverage.xml" + +[tool.pytest.ini_options] +markers = [ + "integration_test: integration/e2e tests which spin up a kind Kubernetes cluster. Requires docker. (deselect with '-m \"not integration_test\"')", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 78fd7177..00000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -markers = - integration_test: integration/e2e tests which spin up a kind Kubernetes cluster. Requires docker. (deselect with '-m "not integration_test"') - diff --git a/requirements.txt b/requirements.txt index eba7998c..2930e345 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ --index-url https://pypi.python.org/simple/ --e .[dev] +-e .[dev,ci] diff --git a/setup.cfg b/setup.cfg index ea7d083e..0f01ba5e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,6 @@ -[egg_info] -# PEP440 compliant developmental version tag https://peps.python.org/pep-0440/#developmental-releases -tag_build=.dev - [aliases] test=pytest -[coverage:html] -directory=build/reports/coverage - -[coverage:xml] -output=build/reports/coverage.xml - [flake8] max-line-length=140 max-complexity=10 diff --git a/setup.py b/setup.py deleted file mode 100755 index 6a0cabfa..00000000 --- a/setup.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 - -# Copyright 2017-2019 The FIAAS Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -from setuptools import setup, find_packages - - -def read(filename): - with open(os.path.join(os.path.dirname(__file__), filename)) as f: - return f.read() - - -GENERIC_REQ = [ - "ConfigArgParse == 0.14.0", - "prometheus_client == 0.7.1", - "PyYAML == 5.1.2", - "pyaml == 19.4.1", - "pinject == 0.14.1", - "decorator < 5.0.0", # 5.0.0 and later drops py2 support (transitive dep from pinject) - "six >= 1.12.0", - "dnspython == 1.16.0", - "k8s == 0.27.3", - "appdirs == 1.4.3", - "requests-toolbelt == 0.10.1", - "backoff == 1.8.0", - "py27hash == 1.1.0", -] - -WEB_REQ = [ - "Flask == 3.0.0", - "flask-talisman >= 1.1.0", - "jinja2 >= 3.0.1", - "markupsafe >= 2.1.3", - "itsdangerous >= 2.1.2", - "werkzeug >= 3.0.1", - "blinker >= 1.7.0", -] - -DEPLOY_REQ = [ - "urllib3 == 1.26.17", - "requests == 2.32.3", - "ipaddress == 1.0.22", # Required by requests for resolving IP address in SSL cert -] - -FLAKE8_REQ = [ - "flake8-print == 3.1.4", - "flake8-comprehensions == 1.4.1", - "pep8-naming == 0.11.1", - "flake8 == 7.0.0", -] - -TESTS_REQ = [ - "pytest-xdist == 3.6.1", - "pytest-sugar == 1.0.0", - "pytest-html == 4.1.1", - "pytest-cov == 5.0.0", - "pytest-helpers-namespace == 2021.12.29", - "pytest == 8.2.0", - "requests-file == 1.4.3", - "callee == 0.3.1", -] - -DEV_TOOLS = [ - "tox==4.14.2", - "virtualenv==20.26.0", - "black ~= 22.0", -] - - -if __name__ == "__main__": - setup( - name="fiaas-deploy-daemon", - author="FINN Team Infrastructure", - author_email="FINN-TechteamInfrastruktur@finn.no", - version="1.0", - packages=find_packages(exclude=("tests",)), - zip_safe=True, - include_package_data=True, - # Requirements - install_requires=GENERIC_REQ + WEB_REQ + DEPLOY_REQ, - setup_requires=["pytest-runner", "wheel", "setuptools_git >= 1.2"], - extras_require={ - "dev": TESTS_REQ + FLAKE8_REQ + DEV_TOOLS, - "ci": DEV_TOOLS, - }, - # Metadata - description="Deploy applications to Kubernetes", - long_description=read("README.md"), - url="https://github.schibsted.io/finn/fiaas-deploy-daemon", - # Entrypoints - entry_points={ - "console_scripts": [ - "fiaas-deploy-daemon = fiaas_deploy_daemon:main", - "fiaas-deploy-daemon-bootstrap = fiaas_deploy_daemon.bootstrap:main", - ] - }, - ) From c7126256117b4dd3bcd2ef63c4bf3bf5f34ee619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Fri, 4 Oct 2024 10:32:38 +0200 Subject: [PATCH 2/4] k8s and requests To use requests 2.32.3 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e498f8a8..f923028b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "decorator < 5.0.0", # 5.0.0 and later drops py2 support (transitive dep from pinject) "six >= 1.12.0", "dnspython == 1.16.0", - "k8s == 0.27.1", + "k8s == 0.27.3", "appdirs == 1.4.3", "requests-toolbelt == 0.10.1", "backoff == 1.8.0", @@ -41,7 +41,7 @@ dependencies = [ "werkzeug >= 3.0.1", "blinker >= 1.7.0", "urllib3 == 1.26.17", - "requests == 2.31.0", + "requests == 2.32.3", "ipaddress == 1.0.22", # Required by requests for resolving IP address in SSL cert ] From 91d2d7eaa048f5622da837aa5453fcf994345a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Fri, 4 Oct 2024 14:04:24 +0200 Subject: [PATCH 3/4] Ensure defaults.yml and html templates are in package Add setuptools_scm to auto-detect tracked data files in included packages https://setuptools.pypa.io/en/latest/userguide/datafiles.html#configuration-options --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f923028b..b9ee7002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,8 @@ [build-system] requires = [ - "setuptools>=61.0", - "wheel", + "setuptools>=61.0", + "setuptools_scm", # required to include data files (defaults.yml and the html templates) in the python packages + "wheel", ] build-backend = "setuptools.build_meta" From 258508a1ae5b323c9d0c98c08fd7ac7240c70826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Ingebrigtsen=20=C3=98vergaard?= Date: Fri, 4 Oct 2024 14:05:42 +0200 Subject: [PATCH 4/4] Use same behaviour as with setup.py for finding packages When using find_packages via pyproject.toml, namespaced packages are automatically included. This means all directories in the repository root is included by default and we don't want that. The ideal solution might be to change the repo structure to src-layout[1], but for now use namespaces=false to use the same behaviour with setup.py find_packages to keep the changeset as small as possible. [1]: https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b9ee7002..c2184125 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ requires = [ ] build-backend = "setuptools.build_meta" - [tool.setuptools.packages.find] -exclude = ["tests"] +exclude = ["tests*"] +namespaces = false # only consider directories with __init__.py as packages (old setuptools setup.py behaviour) [project] name = "fiaas-deploy-daemon"