From 0bdf0c7b1e1792fc36158c5198a7ee22fefe4747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Thu, 7 Nov 2024 09:01:31 +0100 Subject: [PATCH] Fix first run (dry run) --- .github/publish.yaml | 2 +- config.md | 13 ++-- tag_publish/__init__.py | 23 +++++- tag_publish/cli.py | 142 +++++++++++++++++++++++++---------- tag_publish/configuration.py | 22 +----- tag_publish/publish.py | 4 - tag_publish/schema.json | 41 ++++------ 7 files changed, 146 insertions(+), 101 deletions(-) diff --git a/.github/publish.yaml b/.github/publish.yaml index af610f0..020b166 100644 --- a/.github/publish.yaml +++ b/.github/publish.yaml @@ -5,7 +5,7 @@ pypi: - version_tag - version_branch packages: - - path: . + - {} docker: images: - name: camptocamp/tag-publish diff --git a/config.md b/config.md index 7826ef9..9f05c65 100644 --- a/config.md +++ b/config.md @@ -50,14 +50,11 @@ _Tag Publish configuration file_ - **Items** _(string)_ - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. - **Items** _(string)_ -- **`helm`**: Configuration to publish Helm charts on GitHub release. - - **One of** - - _object_: Configuration to publish on Helm charts on GitHub release. - - **`folders`** _(array)_: The folders that will be published. - - **Items** _(string)_ - - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. - - **Items** _(string)_ - - : Must be: `false`. +- **`helm`** _(object)_: Configuration to publish Helm charts on GitHub release. + - **`folders`** _(array)_: The folders that will be published. + - **Items** _(string)_ + - **`versions`** _(array)_: The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script. Default: `["version_tag"]`. + - **Items** _(string)_ - **`version_transform`** _(array)_: A version transformer definition. - **Items** _(object)_ - **`from`** _(string)_: The from regular expression. diff --git a/tag_publish/__init__.py b/tag_publish/__init__.py index 631a86c..a27554e 100644 --- a/tag_publish/__init__.py +++ b/tag_publish/__init__.py @@ -32,10 +32,25 @@ class GH: def __init__(self) -> None: """Initialize the GitHub helper class.""" - token = os.environ["GITHUB_TOKEN"] + token = ( + os.environ["GITHUB_TOKEN"] + if "GITHUB_TOKEN" in os.environ + else subprocess.run( + ["gh", "auth", "token"], check=True, stdout=subprocess.PIPE, encoding="utf-8" + ).stdout.strip() + ) self.auth = github.Auth.Token(token) self.github = github.Github(auth=self.auth) - self.repo = self.github.get_repo(os.environ["GITHUB_REPOSITORY"]) + self.repo = self.github.get_repo( + os.environ["GITHUB_REPOSITORY"] + if "GITHUB_REPOSITORY" in os.environ + else subprocess.run( + ["gh", "repo", "view", "--json", "name,owner", "--jq", '(.owner.login + "/" + .name)'], + check=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ).stdout.strip() + ) self.default_branch = self.repo.default_branch @@ -77,8 +92,8 @@ def get_config(gh: GH) -> tag_publish.configuration.Configuration: Get the configuration, with project and auto detections. """ config: tag_publish.configuration.Configuration = {} - if os.path.exists("ci/config.yaml"): - with open("ci/config.yaml", encoding="utf-8") as open_file: + if os.path.exists(".github/publish.yaml"): + with open(".github/publish.yaml", encoding="utf-8") as open_file: yaml_ = ruamel.yaml.YAML() config = yaml_.load(open_file) diff --git a/tag_publish/cli.py b/tag_publish/cli.py index bfde520..015c094 100644 --- a/tag_publish/cli.py +++ b/tag_publish/cli.py @@ -79,6 +79,8 @@ def main() -> None: parser.add_argument("--branch", help="The branch from which to compute the version") parser.add_argument("--tag", help="The tag from which to compute the version") parser.add_argument("--dry-run", action="store_true", help="Don't do the publish") + parser.add_argument("--dry-run-tag", help="Don't do the publish, on a tag") + parser.add_argument("--dry-run-branch", help="Don't do the publish, on a branch") parser.add_argument( "--type", help="The type of version, if no argument provided auto-determinate, can be: " @@ -87,6 +89,13 @@ def main() -> None: ) args = parser.parse_args() + if args.dry_run_tag is not None: + args.dry_run = True + os.environ["GITHUB_REF"] = f"refs/tags/{args.dry_run_tag}" + if args.dry_run_branch is not None: + args.dry_run = True + os.environ["GITHUB_REF"] = f"refs/heads/{args.dry_run_branch}" + github = tag_publish.GH() config = tag_publish.get_config(github) @@ -173,21 +182,50 @@ def main() -> None: success = True published_payload: list[tag_publish.PublishedPayload] = [] - pypi_config = cast( - tag_publish.configuration.Pypi, - config.get("pypi", {}) if config.get("pypi", False) else {}, + success &= _handle_pypi_publish( + args.group, args.dry_run, config, version, version_type, github, published_payload + ) + success &= _handle_docker_publish( + args.group, + args.dry_run, + args.docke_versions, + args.snyk_version, + config, + version, + version_type, + github, + published_payload, + local, ) + success &= _handle_helm_publish(args.dry_run, config, version, version_type, github, published_payload) + _trigger_dispatch_events(config, published_payload, github) + + if not success: + sys.exit(1) + + +def _handle_pypi_publish( + group: str, + dry_run: bool, + config: tag_publish.configuration.Configuration, + version: str, + version_type: str, + github: tag_publish.GH, + published_payload: list[tag_publish.PublishedPayload], +) -> bool: + success = True + pypi_config = config.get("pypi", {}) if pypi_config: - if pypi_config["packages"]: + if "packages" in pypi_config: tag_publish.lib.oidc.pypi_login() for package in pypi_config["packages"]: - if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == args.group: + if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == group: publish = version_type in pypi_config.get( "versions", tag_publish.configuration.PYPI_VERSIONS_DEFAULT ) path = package.get("path", tag_publish.configuration.PYPI_PACKAGE_PATH_DEFAULT) - if args.dry_run: + if dry_run: print(f"{'Publishing' if publish else 'Checking'} '{path}' to pypi, skipping (dry run)") else: success &= tag_publish.publish.pip(package, version, version_type, publish, github) @@ -199,11 +237,23 @@ def main() -> None: "version_type": version_type, } ) - - docker_config = cast( - tag_publish.configuration.Docker, - config.get("docker", {}) if config.get("docker", False) else {}, - ) + return success + + +def _handle_docker_publish( + group: str, + dry_run: bool, + docker_versions: str, + snyk_version: str, + config: tag_publish.configuration.Configuration, + version: str, + version_type: str, + github: tag_publish.GH, + published_payload: list[tag_publish.PublishedPayload], + local: bool, +) -> bool: + success = True + docker_config = config.get("docker", {}) if docker_config: security_text = "" if local: @@ -232,7 +282,6 @@ def main() -> None: add_latest = True for data in security.data: row_tags = {t.strip() for t in data[alternate_tag_index].split(",") if t.strip()} - print(row_tags) if "latest" in row_tags: print("latest found in ", row_tags) add_latest = False @@ -243,23 +292,23 @@ def main() -> None: images_src: set[str] = set() images_full: list[str] = [] images_snyk: set[str] = set() - versions = args.docker_versions.split(",") if args.docker_versions else [version] + versions = docker_versions.split(",") if docker_versions else [version] for image_conf in docker_config.get("images", []): - if image_conf.get("group", tag_publish.configuration.DOCKER_IMAGE_GROUP_DEFAULT) == args.group: + if image_conf.get("group", tag_publish.configuration.DOCKER_IMAGE_GROUP_DEFAULT) == group: for tag_config in image_conf.get("tags", tag_publish.configuration.DOCKER_IMAGE_TAGS_DEFAULT): tag_src = tag_config.format(version="latest") image_source = f"{image_conf['name']}:{tag_src}" images_src.add(image_source) - tag_snyk = tag_config.format(version=args.snyk_version or version).lower() + tag_snyk = tag_config.format(version=snyk_version or version).lower() image_snyk = f"{image_conf['name']}:{tag_snyk}" # Workaround sine we have the business plan image_snyk = f"{image_conf['name']}_{tag_snyk}" - if not args.dry_run: + if not dry_run: subprocess.run(["docker", "tag", image_source, image_snyk], check=True) images_snyk.add(image_snyk) - if tag_snyk != tag_src and not args.dry_run: + if tag_snyk != tag_src and not dry_run: subprocess.run( [ "docker", @@ -287,7 +336,7 @@ def main() -> None: for alt_tag in [docker_version, *alt_tags] ] - if args.dry_run: + if dry_run: for tag in tags: print( f"Publishing {image_conf['name']}:{tag} to {name}, skipping " @@ -305,7 +354,7 @@ def main() -> None: published_payload, ) - if args.dry_run: + if dry_run: sys.exit(0) snyk_exec, env = tag_publish.snyk_exec() @@ -388,15 +437,21 @@ def main() -> None: if dpkg_config_found: success = False + return success - helm_config = cast( - tag_publish.configuration.HelmConfig, - config.get("helm", {}) if config.get("helm", False) else {}, - ) - if ( - helm_config - and helm_config["folders"] - and version_type in helm_config.get("versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT) + +def _handle_helm_publish( + dry_run: bool, + config: tag_publish.configuration.Configuration, + version: str, + version_type: str, + github: tag_publish.GH, + published_payload: list[tag_publish.PublishedPayload], +) -> bool: + success = True + helm_config = config.get("helm", {}) + if helm_config.get("folders") and version_type in helm_config.get( + "versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT ): application_download.cli.download_application("helm/chart-releaser") @@ -432,19 +487,27 @@ def main() -> None: version = ".".join(versions) for folder in helm_config["folders"]: - token = os.environ["GITHUB_TOKEN"] - success &= tag_publish.publish.helm(folder, version, owner, repo, commit_sha, token) - published_payload.append( - { - "type": "helm", - "path": folder, - "version": version, - "version_type": version_type, - } - ) + if dry_run: + print(f"Publishing '{folder}' to helm, skipping (dry run)") + else: + token = os.environ["GITHUB_TOKEN"] + success &= tag_publish.publish.helm(folder, version, owner, repo, commit_sha, token) + published_payload.append( + { + "type": "helm", + "path": folder, + "version": version, + "version_type": version_type, + } + ) + return success - config = tag_publish.get_config(tag_publish.GH()) +def _trigger_dispatch_events( + config: tag_publish.configuration.Configuration, + published_payload: list[tag_publish.PublishedPayload], + github: tag_publish.GH, +) -> None: for published in published_payload: for dispatch_config in config.get("dispatch", []): repository = dispatch_config.get("repository") @@ -463,9 +526,6 @@ def main() -> None: github_repo = github.repo github_repo.create_repository_dispatch(event_type, published) # type: ignore[arg-type] - if not success: - sys.exit(1) - if __name__ == "__main__": main() diff --git a/tag_publish/configuration.py b/tag_publish/configuration.py index 899b75f..feb69b5 100644 --- a/tag_publish/configuration.py +++ b/tag_publish/configuration.py @@ -38,9 +38,6 @@ class Configuration(TypedDict, total=False): helm. Configuration to publish Helm charts on GitHub release - - Aggregation type: oneOf - Subtype: "HelmConfig" """ dispatch: List["DispatchConfig"] @@ -211,25 +208,14 @@ class DockerRepository(TypedDict, total=False): HELM_VERSIONS_DEFAULT = ["version_tag"] -""" Default value of the field path 'helm config versions' """ - - -Helm = Union["HelmConfig", Literal[False]] -""" -helm. +""" Default value of the field path 'helm versions' """ -Configuration to publish Helm charts on GitHub release -Aggregation type: oneOf -Subtype: "HelmConfig" -""" - - -class HelmConfig(TypedDict, total=False): +class Helm(TypedDict, total=False): """ - helm config. + helm. - Configuration to publish on Helm charts on GitHub release + Configuration to publish Helm charts on GitHub release """ folders: List[str] diff --git a/tag_publish/publish.py b/tag_publish/publish.py index 9a9aa1d..8b9ccc6 100644 --- a/tag_publish/publish.py +++ b/tag_publish/publish.py @@ -79,10 +79,6 @@ def pip( ["pip", "install", *pyproject.get("build-system", {}).get("requires", [])], check=True ) if use_poetry: - freeze = subprocess.run(["pip", "freeze"], check=True, stdout=subprocess.PIPE) - for freeze_line in freeze.stdout.decode("utf-8").split("\n"): - if freeze_line.startswith("poetry-") or freeze_line.startswith("poetry="): - print(freeze_line) env_bash = " ".join([f"{key}={value}" for key, value in env.items()]) print(f"Run in {cwd}: {env_bash} poetry build") sys.stdout.flush() diff --git a/tag_publish/schema.json b/tag_publish/schema.json index 1cf3309..2d3fb3a 100644 --- a/tag_publish/schema.json +++ b/tag_publish/schema.json @@ -162,34 +162,25 @@ "helm": { "title": "helm", "description": "Configuration to publish Helm charts on GitHub release", - "oneOf": [ - { - "title": "helm config", - "description": "Configuration to publish on Helm charts on GitHub release", - "type": "object", - "properties": { - "folders": { - "description": "The folders that will be published", - "type": "array", - "items": { - "type": "string" - } - }, - "versions": { - "title": "helm versions", - "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", - "type": "array", - "default": ["version_tag"], - "items": { - "type": "string" - } - } + "type": "object", + "properties": { + "folders": { + "description": "The folders that will be published", + "type": "array", + "items": { + "type": "string" } }, - { - "const": false + "versions": { + "title": "helm versions", + "description": "The kind or version that should be published, tag, branch or value of the --version argument of the tag-publish script", + "type": "array", + "default": ["version_tag"], + "items": { + "type": "string" + } } - ] + } }, "version_transform": { "title": "Version transform",