From 2b76633bbc89d33e02b511a39877738f3f359b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Fri, 8 Nov 2024 13:44:12 +0100 Subject: [PATCH 1/2] Better readme --- .github/publish.yaml | 2 +- .github/workflows/main.yaml | 2 + README.md | 129 ++++++++++++++++++++++++++++-------- 3 files changed, 105 insertions(+), 28 deletions(-) diff --git a/.github/publish.yaml b/.github/publish.yaml index b239843..7a800b9 100644 --- a/.github/publish.yaml +++ b/.github/publish.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/tag-publish/master/tag_publish/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/tag-publish/0.7.1/tag_publish/schema.json pypi: versions: diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dd39e08..2e88f0f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -10,7 +10,9 @@ on: pull_request: permissions: + # To publish Docker images packages: write + # To publish Python packages id-token: write env: diff --git a/README.md b/README.md index f2c369a..7b31c70 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,41 @@ ## Publishing The main goals of Tag Publish offer the commands to publish the project, -see the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing). +Using a tag, a stabilization branch, a feature branch or a pull request. + +When possible it can do a secret-less publishing, if it's not possible the login should be done before the publishing. + +See the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing). + +## Startup + +Set the permissions: + +```yaml +permissions: + # To publish Docker images on GHCR + packages: write + # To publish Python packages using OIDC + id-token: write + # To publish Helm charts + contents: write +``` + +Install the package in the worklow: + +```yaml +- name: Install tag-publish + run: pip install c2cciutils-publish +``` + +Do the publishing: + +```yaml +- name: Publish + run: tag-publish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` ## New version @@ -39,21 +73,29 @@ See also [GitHub Documentation](https://docs.github.com/en/github/managing-secur ## Configuration +The configuration file is `.github/publish.yaml`, the schema is `https://raw.githubusercontent.com/camptocamp/tag-publish//tag_publish/schema.json`. + ### Dry run Dry run publish: `GITHUB_REF=... c2cciutils-publish --dry-run ...` ### To pypi -The config is like this: +Minimum configuration: ```yaml -versions: - # List of kinds of versions you want to publish, that can be: - # rebuild (specified with --type), - # version_tag, version_branch, feature_branch, feature_tag (for pull request) +pypi: + packages: + - {} ``` +If the file `~/.pypirc` didn't exists we will do a login using OpenId Connect (OIDC), see: +https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi. + +By default the package will be published only on tag, if you want to publish on stabilization branch you should add +a `versions` key with the list of versions you want to publish, that can be: +`rebuild` (specified with --type), `version_tag`, `version_branch`, `feature_branch`, `feature_tag` (for pull request) + It we have a `setup.py` file, we will be in legacy mode: When publishing, the version computed from arguments or `GITHUB_REF` is put in environment variable `VERSION`, thus you should use it in `setup.py`, example: @@ -100,6 +142,8 @@ The OIDC login is recommended because it didn't needs any additional secrets, but it need some configuration on pypi in the package, see the [GitHub Documentation](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi#adding-the-identity-provider-to-pypi). +The required permissions is `id-token: write`. + #### Integration if the package directly in a Docker image To make it working in the `Dockerfile` you should have in the `poetry` stage: @@ -130,39 +174,70 @@ build: ## Build the Docker images ### To Docker registry -The config is like this: +The minimal config is like this: + +```yaml +docker: + images: + - name: camptocamp/tag-publish +``` + +If you want to use the GitHub token to be logged in on ghcr you should set `auto_login` to `True`, the +requires the permissions are `packages: write`. + +With that the image initially named `camptocamp/tag-publish:latest` will be published on GitHub CHCR and on Docker hub. + +The full config is like this: ```yaml -latest: True -images: - - # The base name of the image we want to publish - name: -repository: - : - # The fqdn name of the server if not Docker hub - server: - # List of kinds of versions you want to publish, that can be: rebuild (specified using --type), - # version_tag, version_branch, feature_branch, feature_tag (for pull request) - version: - # List of tags we want to publish interpreted with `format(version=version)` - # e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag - # (that should be built by the application build) is `latest-lite`, and it will be published - # with the tag `1.2.3-lite`. - tags: - # If your images are published by different jobs you can separate them in different groups - # and publish them with `tag-publish --group=` - group: +docker: + auto_login: False + latest: True + images: + - # The base name of the image we want to publish + name: + repository: + : + # The fqdn name of the server if not Docker hub + server: + # List of kinds of versions you want to publish, that can be: rebuild (specified using --type), + # version_tag, version_branch, feature_branch, feature_tag (for pull request) + version: + # List of tags we want to publish interpreted with `format(version=version)` + # e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag + # (that should be built by the application build) is `latest-lite`, and it will be published + # with the tag `1.2.3-lite`. + tags: + # If your images are published by different jobs you can separate them in different groups + # and publish them with `tag-publish --group=` + group: ``` By default, the last line of the `SECURITY.md` file will be published (`docker`) with the tag `latest`. Set `latest` to `False` to disable it. -## Use Renovate to trigger a new build instead of the legacy rebuild +#### Use Renovate to trigger a new build instead of the legacy rebuild If the `ci/dpkg-versions.yaml` or `.github/dpkg-versions.yaml` file is present, the package list will be updated on publishing. The versions will be updated by [GHCI](https://github.com/camptocamp/github-app-geo-project/) application. +### HELM + +The minimal config is like this: + +```yaml +helm: + folders: + - . +``` + +This will publish the `helm` charts in the current folder using [chert releaser](https://github.com/helm/chart-releaser). + +The artifacts will be attached to a GitHub release, and the `index.yaml` file will be updated in the `gh-pages` branch. + +The required permission is `contents: write`. + ## Contributing Install the pre-commit hooks: From 3e53235eaeba6b27a6fd07494a6b2f72711f9e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Fri, 8 Nov 2024 11:16:37 +0100 Subject: [PATCH 2/2] Update applications-download, uses jsonschema-validator --- .github/renovate.json5 | 4 ++ poetry.lock | 10 ++-- pyproject.toml | 3 +- tag_publish/__init__.py | 48 +++++++++++++++---- tag_publish/cli.py | 5 +- ...plications-versions.yaml => versions.yaml} | 0 6 files changed, 53 insertions(+), 17 deletions(-) rename tag_publish/{applications-versions.yaml => versions.yaml} (100%) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 1641bdd..c100993 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -35,6 +35,10 @@ datasourceTemplate: 'python-version', depNameTemplate: 'python', }, + { + fileMatch: ['^tag_publish/versions.yaml$'], + matchStrings: ['(?.*): (?.*) # (?.*)'], + }, ], packageRules: [ /** Automerge the patch, the minor and the dev dependency */ diff --git a/poetry.lock b/poetry.lock index bdbd27c..bc74ec3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,14 +12,14 @@ files = [ ] [[package]] -name = "application-download" -version = "0.0.1.dev3" +name = "applications-download" +version = "0.7.1" description = "Tools used to publish Python packages, Docker images and Helm charts for GitHub tag and branch" optional = false python-versions = ">=3.9" files = [ - {file = "application_download-0.0.1.dev3-py3-none-any.whl", hash = "sha256:89662f68d327ccd39c499947c58861ec0c94f648de471fd173d8df53dae25069"}, - {file = "application_download-0.0.1.dev3.tar.gz", hash = "sha256:bc7d894bab6f87d6e822c717eecdc169489e3dc2a61ce8ab7de26a785b40bb57"}, + {file = "applications_download-0.7.1-py3-none-any.whl", hash = "sha256:cb4c1a686d8b2522b3b3fe4632a8305cef59f31a02dfd7e5588292d58dc8f917"}, + {file = "applications_download-0.7.1.tar.gz", hash = "sha256:b385e3c88fb41fd50121ae67e3c69f58613dc252e501e6aff6aea4c4698b1f98"}, ] [package.dependencies] @@ -2214,4 +2214,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "a30510453c99db72224432d0d8d9a084af11aef2983452005a7fabfa0702868e" +content-hash = "ce93a305b88c95fc0e0250164a73ef4793a6a8b6b724a2b94242717a374f9249" diff --git a/pyproject.toml b/pyproject.toml index bbd0480..9bb44b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,8 @@ twine = "5.1.1" PyYAML = "6.0.2" id = "1.4.0" security-md = "0.2.3" -application-download = "0.0.1.dev3" +applications-download = "0.7.1" +jsonschema-validator-new = "0.1.0" PyGithub = "2.5.0" debian-inspector = "31.1.0" multi-repo-automation = "1.4.1" diff --git a/tag_publish/__init__.py b/tag_publish/__init__.py index b087b57..97de100 100644 --- a/tag_publish/__init__.py +++ b/tag_publish/__init__.py @@ -2,17 +2,21 @@ Tag Publish main module. """ +import json import os.path +import pkgutil import re import subprocess # nosec from re import Match, Pattern -from typing import Any, Optional, TypedDict, cast +from typing import Any, Optional, TypedDict, cast, overload -import application_download.cli +import applications_download import github +import jsonschema_validator import requests import ruamel.yaml import security_md +import yaml import tag_publish.configuration @@ -94,9 +98,14 @@ def get_config(gh: GH) -> tag_publish.configuration.Configuration: """ config: tag_publish.configuration.Configuration = {} if os.path.exists(".github/publish.yaml"): + schema_data = pkgutil.get_data("tag_publish", "schema.json") + assert schema_data is not None + schema = json.loads(schema_data) + with open(".github/publish.yaml", encoding="utf-8") as open_file: yaml_ = ruamel.yaml.YAML() config = yaml_.load(open_file) + jsonschema_validator.validate(".github/publish.yaml", cast(dict[str, Any], config), schema) merge( { @@ -235,17 +244,40 @@ def add_authorization_header(headers: dict[str, str]) -> dict[str, str]: return headers +@overload +def download_application(application_name: str, binary_filename: str) -> str: ... + + +@overload +def download_application(application_name: str) -> None: ... + + +def download_application(application_name: str, binary_filename: Optional[str] = None) -> Optional[str]: + """ + Download the application if necessary, with the included version. + """ + binary_full_filename = ( + os.path.expanduser(os.path.join("~", ".local", "bin", binary_filename)) if binary_filename else None + ) + + if not os.path.exists(binary_full_filename) if binary_full_filename else True: + applications = applications_download.load_applications(None) + versions_data = pkgutil.get_data("tag_publish", "versions.yaml") + assert versions_data is not None + versions = yaml.safe_load(versions_data) + applications_download.download_applications( + applications, {application_name: versions[application_name]} + ) + + return binary_full_filename + + def snyk_exec() -> tuple[str, dict[str, str]]: """Get the Snyk cli executable path.""" env = {**os.environ} env["FORCE_COLOR"] = "true" - snyk_bin = os.path.expanduser(os.path.join("~", ".local", "bin", "snyk")) - if not os.path.exists(snyk_bin): - folder = os.path.expanduser(os.path.join("~", ".config", "application_download")) - if not os.path.exists(folder): - os.makedirs(folder) - application_download.cli.download_application("snyk/cli") + snyk_bin = download_application("snyk/cli", "snyk") if "SNYK_TOKEN" not in env: env["SNYK_TOKEN"] = subprocess.run( diff --git a/tag_publish/cli.py b/tag_publish/cli.py index 7fc8bf5..4720732 100644 --- a/tag_publish/cli.py +++ b/tag_publish/cli.py @@ -15,7 +15,6 @@ from re import Match from typing import Optional, cast -import application_download.cli import security_md import yaml @@ -219,7 +218,7 @@ def _handle_pypi_publish( if "packages" in pypi_config: tag_publish.lib.oidc.pypi_login() - for package in pypi_config["packages"]: + for package in pypi_config.get("packages", []): if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == group: publish = version_type in pypi_config.get( "versions", tag_publish.configuration.PYPI_VERSIONS_DEFAULT @@ -466,7 +465,7 @@ def _handle_helm_publish( if helm_config.get("folders") and version_type in helm_config.get( "versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT ): - application_download.cli.download_application("helm/chart-releaser") + tag_publish.download_application("helm/chart-releaser") owner = github.repo.owner.login repo = github.repo.name diff --git a/tag_publish/applications-versions.yaml b/tag_publish/versions.yaml similarity index 100% rename from tag_publish/applications-versions.yaml rename to tag_publish/versions.yaml