Merge pull request #9 from camptocamp/applications-download

Update applications-download, uses jsonschema-validator

.github/publish.yaml

@@ -1,4 +1,4 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/tag-publish/master/tag_publish/schema.json
+# yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/tag-publish/0.7.1/tag_publish/schema.json
 
 pypi:
   versions:

.github/renovate.json5

@@ -35,6 +35,10 @@
       datasourceTemplate: 'python-version',
       depNameTemplate: 'python',
     },
+    {
+      fileMatch: ['^tag_publish/versions.yaml$'],
+      matchStrings: ['(?<depName>.*): (?<currentValue>.*) # (?<datasource>.*)'],
+    },
   ],
   packageRules: [
     /** Automerge the patch, the minor and the dev dependency */

.github/workflows/main.yaml

@@ -10,7 +10,9 @@ on:
   pull_request:
 
 permissions:
+  # To publish Docker images
   packages: write
+  # To publish Python packages
   id-token: write
 
 env:

README.md

@@ -3,7 +3,41 @@
 ## Publishing
 
 The main goals of Tag Publish offer the commands to publish the project,
-see the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing).
+Using a tag, a stabilization branch, a feature branch or a pull request.
+
+When possible it can do a secret-less publishing, if it's not possible the login should be done before the publishing.
+
+See the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing).
+
+## Startup
+
+Set the permissions:
+
+```yaml
+permissions:
+  # To publish Docker images on GHCR
+  packages: write
+  # To publish Python packages using OIDC
+  id-token: write
+  # To publish Helm charts
+  contents: write
+```
+
+Install the package in the worklow:
+
+```yaml
+- name: Install tag-publish
+  run: pip install c2cciutils-publish
+```
+
+Do the publishing:
+
+```yaml
+- name: Publish
+  run: tag-publish
+  env:
+    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+```
 
 ## New version
 
@@ -39,21 +73,29 @@ See also [GitHub Documentation](https://docs.github.com/en/github/managing-secur
 
 ## Configuration
 
+The configuration file is `.github/publish.yaml`, the schema is `https://raw.githubusercontent.com/camptocamp/tag-publish/<version>/tag_publish/schema.json`.
+
 ### Dry run
 
 Dry run publish: `GITHUB_REF=... c2cciutils-publish --dry-run ...`
 
 ### To pypi
 
-The config is like this:
+Minimum configuration:
 
 ```yaml
-versions:
-  # List of kinds of versions you want to publish, that can be:
-  # rebuild (specified with --type),
-  # version_tag, version_branch, feature_branch, feature_tag (for pull request)
+pypi:
+  packages:
+    - {}
 ```
 
+If the file `~/.pypirc` didn't exists we will do a login using OpenId Connect (OIDC), see:
+https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi.
+
+By default the package will be published only on tag, if you want to publish on stabilization branch you should add
+a `versions` key with the list of versions you want to publish, that can be:
+`rebuild` (specified with --type), `version_tag`, `version_branch`, `feature_branch`, `feature_tag` (for pull request)
+
 It we have a `setup.py` file, we will be in legacy mode:
 When publishing, the version computed from arguments or `GITHUB_REF` is put in environment variable `VERSION`, thus you should use it in `setup.py`, example:
 
@@ -100,6 +142,8 @@ The OIDC login is recommended because it didn't needs any additional secrets,
 but it need some configuration on pypi in the package,
 see the [GitHub Documentation](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi#adding-the-identity-provider-to-pypi).
 
+The required permissions is `id-token: write`.
+
 #### Integration if the package directly in a Docker image
 
 To make it working in the `Dockerfile` you should have in the `poetry` stage:
@@ -130,39 +174,70 @@ build: ## Build the Docker images
 
 ### To Docker registry
 
-The config is like this:
+The minimal config is like this:
+
+```yaml
+docker:
+  images:
+    - name: camptocamp/tag-publish
+```
+
+If you want to use the GitHub token to be logged in on ghcr you should set `auto_login` to `True`, the
+requires the permissions are `packages: write`.
+
+With that the image initially named `camptocamp/tag-publish:latest` will be published on GitHub CHCR and on Docker hub.
+
+The full config is like this:
 
 ```yaml
-latest: True
-images:
-  - # The base name of the image we want to publish
-    name:
-repository:
-  <internal_name>:
-    # The fqdn name of the server if not Docker hub
-    server:
-    # List of kinds of versions you want to publish, that can be: rebuild (specified using --type),
-    # version_tag, version_branch, feature_branch, feature_tag (for pull request)
-    version:
-    # List of tags we want to publish interpreted with `format(version=version)`
-    # e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag
-    # (that should be built by the application build) is `latest-lite`, and it will be published
-    # with the tag `1.2.3-lite`.
-    tags:
-    # If your images are published by different jobs you can separate them in different groups
-    # and publish them with `tag-publish --group=<group>`
-    group:
+docker:
+  auto_login: False
+  latest: True
+  images:
+    - # The base name of the image we want to publish
+      name:
+  repository:
+    <internal_name>:
+      # The fqdn name of the server if not Docker hub
+      server:
+      # List of kinds of versions you want to publish, that can be: rebuild (specified using --type),
+      # version_tag, version_branch, feature_branch, feature_tag (for pull request)
+      version:
+      # List of tags we want to publish interpreted with `format(version=version)`
+      # e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag
+      # (that should be built by the application build) is `latest-lite`, and it will be published
+      # with the tag `1.2.3-lite`.
+      tags:
+      # If your images are published by different jobs you can separate them in different groups
+      # and publish them with `tag-publish --group=<group>`
+      group:
 ```
 
 By default, the last line of the `SECURITY.md` file will be published (`docker`) with the tag
 `latest`. Set `latest` to `False` to disable it.
 
-## Use Renovate to trigger a new build instead of the legacy rebuild
+#### Use Renovate to trigger a new build instead of the legacy rebuild
 
 If the `ci/dpkg-versions.yaml` or `.github/dpkg-versions.yaml` file is present, the package list will be updated on publishing.
 
 The versions will be updated by [GHCI](https://github.com/camptocamp/github-app-geo-project/) application.
 
+### HELM
+
+The minimal config is like this:
+
+```yaml
+helm:
+  folders:
+    - .
+```
+
+This will publish the `helm` charts in the current folder using [chert releaser](https://github.com/helm/chart-releaser).
+
+The artifacts will be attached to a GitHub release, and the `index.yaml` file will be updated in the `gh-pages` branch.
+
+The required permission is `contents: write`.
+
 ## Contributing
 
 Install the pre-commit hooks:

pyproject.toml

@@ -66,7 +66,8 @@ twine = "5.1.1"
 PyYAML = "6.0.2"
 id = "1.4.0"
 security-md = "0.2.3"
-application-download = "0.0.1.dev3"
+applications-download = "0.7.1"
+jsonschema-validator-new = "0.1.0"
 PyGithub = "2.5.0"
 debian-inspector = "31.1.0"
 multi-repo-automation = "1.4.1"

tag_publish/__init__.py

@@ -2,17 +2,21 @@
 Tag Publish main module.
 """
 
+import json
 import os.path
+import pkgutil
 import re
 import subprocess  # nosec
 from re import Match, Pattern
-from typing import Any, Optional, TypedDict, cast
+from typing import Any, Optional, TypedDict, cast, overload
 
-import application_download.cli
+import applications_download
 import github
+import jsonschema_validator
 import requests
 import ruamel.yaml
 import security_md
+import yaml
 
 import tag_publish.configuration
 
@@ -94,9 +98,14 @@ def get_config(gh: GH) -> tag_publish.configuration.Configuration:
     """
     config: tag_publish.configuration.Configuration = {}
     if os.path.exists(".github/publish.yaml"):
+        schema_data = pkgutil.get_data("tag_publish", "schema.json")
+        assert schema_data is not None
+        schema = json.loads(schema_data)
+
         with open(".github/publish.yaml", encoding="utf-8") as open_file:
             yaml_ = ruamel.yaml.YAML()
             config = yaml_.load(open_file)
+            jsonschema_validator.validate(".github/publish.yaml", cast(dict[str, Any], config), schema)
 
     merge(
         {
@@ -235,17 +244,40 @@ def add_authorization_header(headers: dict[str, str]) -> dict[str, str]:
         return headers
 
 
+@overload
+def download_application(application_name: str, binary_filename: str) -> str: ...
+
+
+@overload
+def download_application(application_name: str) -> None: ...
+
+
+def download_application(application_name: str, binary_filename: Optional[str] = None) -> Optional[str]:
+    """
+    Download the application if necessary, with the included version.
+    """
+    binary_full_filename = (
+        os.path.expanduser(os.path.join("~", ".local", "bin", binary_filename)) if binary_filename else None
+    )
+
+    if not os.path.exists(binary_full_filename) if binary_full_filename else True:
+        applications = applications_download.load_applications(None)
+        versions_data = pkgutil.get_data("tag_publish", "versions.yaml")
+        assert versions_data is not None
+        versions = yaml.safe_load(versions_data)
+        applications_download.download_applications(
+            applications, {application_name: versions[application_name]}
+        )
+
+    return binary_full_filename
+
+
 def snyk_exec() -> tuple[str, dict[str, str]]:
     """Get the Snyk cli executable path."""
     env = {**os.environ}
     env["FORCE_COLOR"] = "true"
-    snyk_bin = os.path.expanduser(os.path.join("~", ".local", "bin", "snyk"))
 
-    if not os.path.exists(snyk_bin):
-        folder = os.path.expanduser(os.path.join("~", ".config", "application_download"))
-        if not os.path.exists(folder):
-            os.makedirs(folder)
-        application_download.cli.download_application("snyk/cli")
+    snyk_bin = download_application("snyk/cli", "snyk")
 
     if "SNYK_TOKEN" not in env:
         env["SNYK_TOKEN"] = subprocess.run(

tag_publish/cli.py

@@ -15,7 +15,6 @@
 from re import Match
 from typing import Optional, cast
 
-import application_download.cli
 import security_md
 import yaml
 
@@ -219,7 +218,7 @@ def _handle_pypi_publish(
         if "packages" in pypi_config:
             tag_publish.lib.oidc.pypi_login()
 
-        for package in pypi_config["packages"]:
+        for package in pypi_config.get("packages", []):
             if package.get("group", tag_publish.configuration.PIP_PACKAGE_GROUP_DEFAULT) == group:
                 publish = version_type in pypi_config.get(
                     "versions", tag_publish.configuration.PYPI_VERSIONS_DEFAULT
@@ -466,7 +465,7 @@ def _handle_helm_publish(
     if helm_config.get("folders") and version_type in helm_config.get(
         "versions", tag_publish.configuration.HELM_VERSIONS_DEFAULT
     ):
-        application_download.cli.download_application("helm/chart-releaser")
+        tag_publish.download_application("helm/chart-releaser")
 
         owner = github.repo.owner.login
         repo = github.repo.name