From 8eb26c2558fd0a83b5c0b99b50a15a61963414fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 7 Nov 2024 09:01:31 +0100 Subject: [PATCH] Fix first run (dry run) --- .github/publish.yaml | 3 +- .github/workflows/repository-dispatch.yaml | 6 +- config.md | 16 +- tag_publish/__init__.py | 30 ++- tag_publish/applications-versions.yaml | 1 + tag_publish/cli.py | 241 ++++++++++++++------- tag_publish/configuration.py | 45 ++-- tag_publish/publish.py | 9 +- tag_publish/schema.json | 53 +++-- 9 files changed, 245 insertions(+), 159 deletions(-) diff --git a/.github/publish.yaml b/.github/publish.yaml index af610f0..f563213 100644 --- a/.github/publish.yaml +++ b/.github/publish.yaml @@ -5,7 +5,8 @@ pypi: - version_tag - version_branch packages: - - path: . + - {} docker: + auto_login: true images: - name: camptocamp/tag-publish diff --git a/.github/workflows/repository-dispatch.yaml b/.github/workflows/repository-dispatch.yaml index 4faeec4..f6a0dc6 100644 --- a/.github/workflows/repository-dispatch.yaml +++ b/.github/workflows/repository-dispatch.yaml @@ -10,8 +10,8 @@ on: required: true name: description: The package name - path: - description: The package path + folder: + description: The package folder version: description: The package version tag: @@ -33,7 +33,7 @@ jobs: run: | echo "Event type: ${{ github.event.client_payload.type }}" echo "Package name: ${{ github.event.client_payload.name }}" - echo "Package path: ${{ github.event.client_payload.path }}" + echo "Package folder: ${{ github.event.client_payload.folder }}" echo "Package version: ${{ github.event.client_payload.version }}" echo "Package tag: ${{ github.event.client_payload.tag }}" echo "Repository: ${{ github.event.client_payload.repository }}" diff --git a/config.md b/config.md index 7826ef9..dccd6bb 100644 --- a/config.md +++ b/config.md @@ -30,6 +30,7 @@ _Tag Publish configuration file_ - **`server`** _(string)_: The server URL. - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag", "version_branch", "rebuild", "feature_branch"]`. - **Items** _(string)_ + - **`auto_login`** _(boolean)_: Auto login to the GitHub Docker registry. Default: `false`. - **`snyk`** _(object)_: Checks the published images with Snyk. - **`monitor_args`**: The arguments to pass to the Snyk container monitor command. Default: `["--app-vulns"]`. - **One of** @@ -45,19 +46,16 @@ _Tag Publish configuration file_ - **`packages`** _(array)_: The configuration of packages that will be published. - **Items** _(object)_: The configuration of package that will be published. - **`group`** _(string)_: The image is in the group, should be used with the --group option of tag-publish script. Default: `"default"`. - - **`path`** _(string)_: The path of the pypi package. Default: `"."`. + - **`folder`** _(string)_: The folder of the pypi package. Default: `"."`. - **`build_command`** _(array)_: The command used to do the build. - **Items** _(string)_ - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. - **Items** _(string)_ -- **`helm`**: Configuration to publish Helm charts on GitHub release. - - **One of** - - _object_: Configuration to publish on Helm charts on GitHub release. - - **`folders`** _(array)_: The folders that will be published. - - **Items** _(string)_ - - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. - - **Items** _(string)_ - - : Must be: `false`. +- **`helm`** _(object)_: Configuration to publish Helm charts on GitHub release. + - **`folders`** _(array)_: The folders that will be published. + - **Items** _(string)_ + - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. + - **Items** _(string)_ - **`version_transform`** _(array)_: A version transformer definition. - **Items** _(object)_ - **`from`** _(string)_: The from regular expression. diff --git a/tag_publish/__init__.py b/tag_publish/__init__.py index 631a86c..2d4e8e0 100644 --- a/tag_publish/__init__.py +++ b/tag_publish/__init__.py @@ -8,6 +8,7 @@ from re import Match, Pattern from typing import Any, Optional, TypedDict, cast +import application_download.cli import github import requests import ruamel.yaml @@ -32,10 +33,25 @@ class GH: def __init__(self) -> None: """Initialize the GitHub helper class.""" - token = os.environ["GITHUB_TOKEN"] + token = ( + os.environ["GITHUB_TOKEN"] + if "GITHUB_TOKEN" in os.environ + else subprocess.run( + ["gh", "auth", "token"], check=True, stdout=subprocess.PIPE, encoding="utf-8" + ).stdout.strip() + ) self.auth = github.Auth.Token(token) self.github = github.Github(auth=self.auth) - self.repo = self.github.get_repo(os.environ["GITHUB_REPOSITORY"]) + self.repo = self.github.get_repo( + os.environ["GITHUB_REPOSITORY"] + if "GITHUB_REPOSITORY" in os.environ + else subprocess.run( + ["gh", "repo", "view", "--json", "name,owner", "--jq", '(.owner.login + "/" + .name)'], + check=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ).stdout.strip() + ) self.default_branch = self.repo.default_branch @@ -77,8 +93,8 @@ def get_config(gh: GH) -> tag_publish.configuration.Configuration: Get the configuration, with project and auto detections. """ config: tag_publish.configuration.Configuration = {} - if os.path.exists("ci/config.yaml"): - with open("ci/config.yaml", encoding="utf-8") as open_file: + if os.path.exists(".github/publish.yaml"): + with open(".github/publish.yaml", encoding="utf-8") as open_file: yaml_ = ruamel.yaml.YAML() config = yaml_.load(open_file) @@ -224,6 +240,10 @@ def snyk_exec() -> tuple[str, dict[str, str]]: env = {**os.environ} env["FORCE_COLOR"] = "true" snyk_bin = os.path.expanduser(os.path.join("~", ".local", "bin", "snyk")) + + if not os.path.exists(snyk_bin): + application_download.cli.download_application("snyk") + if "SNYK_ORG" in env: subprocess.run([snyk_bin, "config", "set", f"org={env['SNYK_ORG']}"], check=True, env=env) @@ -237,7 +257,7 @@ class PublishedPayload(TypedDict, total=False): type: str name: str - path: str + folder: str version: str tag: str repository: str diff --git a/tag_publish/applications-versions.yaml b/tag_publish/applications-versions.yaml index 1ee744e..307a97f 100644 --- a/tag_publish/applications-versions.yaml +++ b/tag_publish/applications-versions.yaml @@ -1,2 +1,3 @@ # https://docs.renovatebot.com/modules/datasource/#github-releases-datasource helm/chart-releaser: v1.6.1 # github-releases +snyk/cli: v1.1293.1 # github-releases diff --git a/tag_publish/cli.py b/tag_publish/cli.py index bfde520..34df479 100644 --- a/tag_publish/cli.py +++ b/tag_publish/cli.py @@ -79,6 +79,8 @@ def main() -> None: parser.add_argument("--branch", help="The branch from which to compute the version") parser.add_argument("--tag", help="The tag from which to compute the version") parser.add_argument("--dry-run", action="store_true", help="Don't do the publish") + parser.add_argument("--dry-run-tag", help="Don't do the publish, on a tag") + parser.add_argument("--dry-run-branch", help="Don't do the publish, on a branch") parser.add_argument( "--type", help="The type of version, if no argument provided auto-determinate, can be: " @@ -87,6 +89,13 @@ def main() -> None: ) args = parser.parse_args() + if args.dry_run_tag is not None: + args.dry_run = True + os.environ["GITHUB_REF"] = f"refs/tags/{args.dry_run_tag}" + if args.dry_run_branch is not None: + args.dry_run = True + os.environ["GITHUB_REF"] = f"refs/heads/{args.dry_run_branch}" + github = tag_publish.GH() config = tag_publish.get_config(github) @@ -173,38 +182,90 @@ def main() -> None: success = True published_payload: list[tag_publish.PublishedPayload] = [] - pypi_config = cast( - tag_publish.configuration.Pypi, - config.get("pypi", {}) if config.get("pypi", False) else {}, + success &= _handle_pypi_publish( + args.group, args.dry_run, config, version, version_type, github, published_payload + ) + success &= _handle_docker_publish( + args.group, + args.dry_run, + args.docker_versions, + args.snyk_version, + config, + version, + version_type, + github, + published_payload, + local, ) + success &= _handle_helm_publish(args.dry_run, config, version, version_type, github, published_payload) + _trigger_dispatch_events(config, published_payload, github) + + if not success: + sys.exit(1) + + +def _handle_pypi_publish( + group: str, + dry_run: bool, + config: tag_publish.configuration.Configuration, + version: str, + version_type: str, + github: tag_publish.GH, + published_payload: list[tag_publish.PublishedPayload], +) -> bool: + success = True + pypi_config = config.get("pypi", {}) if pypi_config: - if pypi_config["packages"]: + if "packages" in pypi_config: tag_publish.lib.oidc.pypi_login() for package in pypi_config["packages"]: - if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == args.group: + if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == group: publish = version_type in pypi_config.get( "versions", tag_publish.configuration.PYPI_VERSIONS_DEFAULT ) - path = package.get("path", tag_publish.configuration.PYPI_PACKAGE_PATH_DEFAULT) - if args.dry_run: - print(f"{'Publishing' if publish else 'Checking'} '{path}' to pypi, skipping (dry run)") + folder = package.get("folder", tag_publish.configuration.PYPI_PACKAGE_FOLDER_DEFAULT) + if dry_run: + print(f"{'Publishing' if publish else 'Checking'} '{folder}' to pypi, skipping (dry run)") else: success &= tag_publish.publish.pip(package, version, version_type, publish, github) published_payload.append( { "type": "pypi", - "path": "path", + "folder": folder, "version": version, "version_type": version_type, } ) - - docker_config = cast( - tag_publish.configuration.Docker, - config.get("docker", {}) if config.get("docker", False) else {}, - ) + return success + + +def _handle_docker_publish( + group: str, + dry_run: bool, + docker_versions: str, + snyk_version: str, + config: tag_publish.configuration.Configuration, + version: str, + version_type: str, + github: tag_publish.GH, + published_payload: list[tag_publish.PublishedPayload], + local: bool, +) -> bool: + success = True + docker_config = config.get("docker", {}) if docker_config: + if docker_config.get("auto_login", tag_publish.configuration.DOCKER_AUTO_LOGIN_DEFAULT): + subprocess.run( + [ + "docker", + "login", + "ghcr.io", + "--username=github", + f"--password={os.environ['GITHUB_TOKEN']}", + ], + check=True, + ) security_text = "" if local: with open("SECURITY.md", encoding="utf-8") as security_file: @@ -232,7 +293,6 @@ def main() -> None: add_latest = True for data in security.data: row_tags = {t.strip() for t in data[alternate_tag_index].split(",") if t.strip()} - print(row_tags) if "latest" in row_tags: print("latest found in ", row_tags) add_latest = False @@ -243,23 +303,23 @@ def main() -> None: images_src: set[str] = set() images_full: list[str] = [] images_snyk: set[str] = set() - versions = args.docker_versions.split(",") if args.docker_versions else [version] + versions = docker_versions.split(",") if docker_versions else [version] for image_conf in docker_config.get("images", []): - if image_conf.get("group", tag_publish.configuration.DOCKER_IMAGE_GROUP_DEFAULT) == args.group: + if image_conf.get("group", tag_publish.configuration.DOCKER_IMAGE_GROUP_DEFAULT) == group: for tag_config in image_conf.get("tags", tag_publish.configuration.DOCKER_IMAGE_TAGS_DEFAULT): tag_src = tag_config.format(version="latest") image_source = f"{image_conf['name']}:{tag_src}" images_src.add(image_source) - tag_snyk = tag_config.format(version=args.snyk_version or version).lower() + tag_snyk = tag_config.format(version=snyk_version or version).lower() image_snyk = f"{image_conf['name']}:{tag_snyk}" # Workaround sine we have the business plan image_snyk = f"{image_conf['name']}_{tag_snyk}" - if not args.dry_run: + if not dry_run: subprocess.run(["docker", "tag", image_source, image_snyk], check=True) images_snyk.add(image_snyk) - if tag_snyk != tag_src and not args.dry_run: + if tag_snyk != tag_src and not dry_run: subprocess.run( [ "docker", @@ -287,7 +347,7 @@ def main() -> None: for alt_tag in [docker_version, *alt_tags] ] - if args.dry_run: + if dry_run: for tag in tags: print( f"Publishing {image_conf['name']}:{tag} to {name}, skipping " @@ -305,52 +365,54 @@ def main() -> None: published_payload, ) - if args.dry_run: + if dry_run: sys.exit(0) - snyk_exec, env = tag_publish.snyk_exec() - for image in images_snyk: - print(f"::group::Snyk check {image}") - sys.stdout.flush() - sys.stderr.flush() - try: - if version_type in ("version_branch", "version_tag"): - monitor_args = docker_config.get("snyk", {}).get( - "monitor_args", - tag_publish.configuration.DOCKER_SNYK_MONITOR_ARGS_DEFAULT, + has_gopass = subprocess.run(["gopass", "--version"]).returncode == 0 # nosec # pylint: disable=subprocess-run-check + if "SNYK_TOKEN" in os.environ or has_gopass: + snyk_exec, env = tag_publish.snyk_exec() + for image in images_snyk: + print(f"::group::Snyk check {image}") + sys.stdout.flush() + sys.stderr.flush() + try: + if version_type in ("version_branch", "version_tag"): + monitor_args = docker_config.get("snyk", {}).get( + "monitor_args", + tag_publish.configuration.DOCKER_SNYK_MONITOR_ARGS_DEFAULT, + ) + if monitor_args is not False: + subprocess.run( # pylint: disable=subprocess-run-check + [ + snyk_exec, + "container", + "monitor", + *monitor_args, + # Available only on the business plan + # f"--project-tags=tag={image.split(':')[-1]}", + image, + ], + env=env, + ) + test_args = docker_config.get("snyk", {}).get( + "test_args", tag_publish.configuration.DOCKER_SNYK_TEST_ARGS_DEFAULT ) - if monitor_args is not False: - subprocess.run( # pylint: disable=subprocess-run-check - [ - snyk_exec, - "container", - "monitor", - *monitor_args, - # Available only on the business plan - # f"--project-tags=tag={image.split(':')[-1]}", - image, - ], + snyk_error = False + if test_args is not False: + proc = subprocess.run( + [snyk_exec, "container", "test", *test_args, image], + check=False, env=env, ) - test_args = docker_config.get("snyk", {}).get( - "test_args", tag_publish.configuration.DOCKER_SNYK_TEST_ARGS_DEFAULT - ) - snyk_error = False - if test_args is not False: - proc = subprocess.run( - [snyk_exec, "container", "test", *test_args, image], - check=False, - env=env, - ) - if proc.returncode != 0: - snyk_error = True - print("::endgroup::") - if snyk_error: - print("::error::Critical vulnerability found by Snyk in the published image.") - except subprocess.CalledProcessError as exception: - print(f"Error: {exception}") - print("::endgroup::") - print("::error::With error") + if proc.returncode != 0: + snyk_error = True + print("::endgroup::") + if snyk_error: + print("::error::Critical vulnerability found by Snyk in the published image.") + except subprocess.CalledProcessError as exception: + print(f"Error: {exception}") + print("::endgroup::") + print("::error::With error") versions_config, dpkg_config_found = tag_publish.lib.docker.get_versions_config() dpkg_success = True @@ -388,15 +450,21 @@ def main() -> None: if dpkg_config_found: success = False + return success - helm_config = cast( - tag_publish.configuration.HelmConfig, - config.get("helm", {}) if config.get("helm", False) else {}, - ) - if ( - helm_config - and helm_config["folders"] - and version_type in helm_config.get("versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT) + +def _handle_helm_publish( + dry_run: bool, + config: tag_publish.configuration.Configuration, + version: str, + version_type: str, + github: tag_publish.GH, + published_payload: list[tag_publish.PublishedPayload], +) -> bool: + success = True + helm_config = config.get("helm", {}) + if helm_config.get("folders") and version_type in helm_config.get( + "versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT ): application_download.cli.download_application("helm/chart-releaser") @@ -432,19 +500,27 @@ def main() -> None: version = ".".join(versions) for folder in helm_config["folders"]: - token = os.environ["GITHUB_TOKEN"] - success &= tag_publish.publish.helm(folder, version, owner, repo, commit_sha, token) - published_payload.append( - { - "type": "helm", - "path": folder, - "version": version, - "version_type": version_type, - } - ) + if dry_run: + print(f"Publishing '{folder}' to helm, skipping (dry run)") + else: + token = os.environ["GITHUB_TOKEN"] + success &= tag_publish.publish.helm(folder, version, owner, repo, commit_sha, token) + published_payload.append( + { + "type": "helm", + "folder": folder, + "version": version, + "version_type": version_type, + } + ) + return success - config = tag_publish.get_config(tag_publish.GH()) +def _trigger_dispatch_events( + config: tag_publish.configuration.Configuration, + published_payload: list[tag_publish.PublishedPayload], + github: tag_publish.GH, +) -> None: for published in published_payload: for dispatch_config in config.get("dispatch", []): repository = dispatch_config.get("repository") @@ -463,9 +539,6 @@ def main() -> None: github_repo = github.repo github_repo.create_repository_dispatch(event_type, published) # type: ignore[arg-type] - if not success: - sys.exit(1) - if __name__ == "__main__": main() diff --git a/tag_publish/configuration.py b/tag_publish/configuration.py index 899b75f..0891cc7 100644 --- a/tag_publish/configuration.py +++ b/tag_publish/configuration.py @@ -38,9 +38,6 @@ class Configuration(TypedDict, total=False): helm. Configuration to publish Helm charts on GitHub release - - Aggregation type: oneOf - Subtype: "HelmConfig" """ dispatch: List["DispatchConfig"] @@ -68,6 +65,10 @@ class Configuration(TypedDict, total=False): """ Default value of the field path 'dispatch config repository' """ +DOCKER_AUTO_LOGIN_DEFAULT = False +""" Default value of the field path 'Docker auto_login' """ + + DOCKER_IMAGE_GROUP_DEFAULT = "default" """ Default value of the field path 'Docker image group' """ @@ -160,6 +161,15 @@ class Docker(TypedDict, total=False): - rebuild """ + auto_login: bool + """ + Docker auto login. + + Auto login to the GitHub Docker registry + + default: False + """ + snyk: "_DockerSnyk" """ Checks the published images with Snyk """ @@ -211,25 +221,14 @@ class DockerRepository(TypedDict, total=False): HELM_VERSIONS_DEFAULT = ["version_tag"] -""" Default value of the field path 'helm config versions' """ +""" Default value of the field path 'helm versions' """ -Helm = Union["HelmConfig", Literal[False]] -""" -helm. - -Configuration to publish Helm charts on GitHub release - -Aggregation type: oneOf -Subtype: "HelmConfig" -""" - - -class HelmConfig(TypedDict, total=False): +class Helm(TypedDict, total=False): """ - helm config. + helm. - Configuration to publish on Helm charts on GitHub release + Configuration to publish Helm charts on GitHub release """ folders: List[str] @@ -250,8 +249,8 @@ class HelmConfig(TypedDict, total=False): """ Default value of the field path 'pypi package group' """ -PYPI_PACKAGE_PATH_DEFAULT = "." -""" Default value of the field path 'pypi package path' """ +PYPI_PACKAGE_FOLDER_DEFAULT = "." +""" Default value of the field path 'pypi package folder' """ PYPI_VERSIONS_DEFAULT = ["version_tag"] @@ -295,11 +294,11 @@ class PypiPackage(TypedDict, total=False): default: default """ - path: str + folder: str """ - pypi package path. + pypi package folder. - The path of the pypi package + The folder of the pypi package default: . """ diff --git a/tag_publish/publish.py b/tag_publish/publish.py index 9a9aa1d..c5e434f 100644 --- a/tag_publish/publish.py +++ b/tag_publish/publish.py @@ -35,7 +35,8 @@ def pip( github: The GitHub helper """ - print(f"::group::{'Publishing' if publish else 'Checking'} '{package.get('path')}' to pypi") + folder = package.get("folder", tag_publish.configuration.PYPI_PACKAGE_FOLDER_DEFAULT) + print(f"::group::{'Publishing' if publish else 'Checking'} '{folder}' to pypi") sys.stdout.flush() sys.stderr.flush() @@ -47,7 +48,7 @@ def pip( is_master = default_branch == version env["IS_MASTER"] = "TRUE" if is_master else "FALSE" - cwd = os.path.abspath(package.get("path", ".")) + cwd = os.path.abspath(folder) dist = os.path.join(cwd, "dist") if not os.path.exists(dist): @@ -79,10 +80,6 @@ def pip( ["pip", "install", *pyproject.get("build-system", {}).get("requires", [])], check=True ) if use_poetry: - freeze = subprocess.run(["pip", "freeze"], check=True, stdout=subprocess.PIPE) - for freeze_line in freeze.stdout.decode("utf-8").split("\n"): - if freeze_line.startswith("poetry-") or freeze_line.startswith("poetry="): - print(freeze_line) env_bash = " ".join([f"{key}={value}" for key, value in env.items()]) print(f"Run in {cwd}: {env_bash} poetry build") sys.stdout.flush() diff --git a/tag_publish/schema.json b/tag_publish/schema.json index 1cf3309..52cbad7 100644 --- a/tag_publish/schema.json +++ b/tag_publish/schema.json @@ -77,6 +77,12 @@ } } }, + "auto_login": { + "title": "Docker auto login", + "description": "Auto login to the GitHub Docker registry", + "type": "boolean", + "default": false + }, "snyk": { "description": "Checks the published images with Snyk", "type": "object", @@ -132,9 +138,9 @@ "default": "default", "type": "string" }, - "path": { - "title": "pypi package path", - "description": "The path of the pypi package", + "folder": { + "title": "pypi package folder", + "description": "The folder of the pypi package", "type": "string", "default": "." }, @@ -162,34 +168,25 @@ "helm": { "title": "helm", "description": "Configuration to publish Helm charts on GitHub release", - "oneOf": [ - { - "title": "helm config", - "description": "Configuration to publish on Helm charts on GitHub release", - "type": "object", - "properties": { - "folders": { - "description": "The folders that will be published", - "type": "array", - "items": { - "type": "string" - } - }, - "versions": { - "title": "helm versions", - "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", - "type": "array", - "default": ["version_tag"], - "items": { - "type": "string" - } - } + "type": "object", + "properties": { + "folders": { + "description": "The folders that will be published", + "type": "array", + "items": { + "type": "string" } }, - { - "const": false + "versions": { + "title": "helm versions", + "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", + "type": "array", + "default": ["version_tag"], + "items": { + "type": "string" + } } - ] + } }, "version_transform": { "title": "Version transform",