[PKG-7597] Publish packages with attestation bundle (#7)

buildkite-plugins · Sep 17, 2024 · 01ccd4d · 01ccd4d
1 parent 060de87
commit 01ccd4d
Show file tree

Hide file tree

Showing 10 changed files with 264 additions and 39 deletions.
diff --git a/README.md b/README.md
@@ -1,9 +1,37 @@
 # Publish to Packages [![Build status](https://badge.buildkite.com/8dff045aea2a2227a4387e77941af1177230066dc459982c67.svg)](https://buildkite.com/buildkite/plugins-publish-to-packages)
 
-A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that publishes [build artifacts](https://buildkite.com/docs/pipelines/artifacts) to [Buildkite Packages](https://buildkite.com/packages).
+A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that publishes [build artifacts](https://buildkite.com/docs/pipelines/artifacts) and attestations to [Buildkite Packages](https://buildkite.com/packages).
 
 This plugin authenticates with Buildkite Packages using an [Agent OIDC token](https://buildkite.com/docs/agent/v3/cli-oidc), so your registry needs to be configured with a suitable [OIDC policy](https://buildkite.com/docs/packages/security/oidc#define-an-oidc-policy-for-a-registry).
 
+## Quick Start
+
+### Minimal
+
+```yaml
+steps:
+  - label: "Publish Gem"
+    plugins:
+      - publish-to-packages#v2.0.0:
+          artifacts: "awesome-logger-*.gem"
+          registry: "acme-corp/awesome-logger"
+```
+
+### The Works
+
+```yaml
+steps:
+  - label: "Publish Gem"
+    plugins:
+      - publish-to-packages#v2.0.0:
+          artifacts: "awesome-logger-*.gem"
+          registry: "acme-corp/awesome-logger"
+          attestations: # optional
+            - "gem-build.attestation.json"
+            - "gem-package.attestation.json"
+          artifact_build_id: "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}" # optional
+```
+
 ## Options
 
 #### `artifacts` (string, required)
@@ -17,6 +45,14 @@ Buildkite Packages registry to publish to.
 - Full format is `<organization>/<registry_name>` (e.g. `acme-corp/awesome-logger`).
 - `<organization>` defaults to your Buildkite organization if omitted (e.g. `awesome-logger`).
 
+#### `attestations` (string or list of strings, optional)
+
+One or more attestations from artifact storage to publish along with each package created from `artifacts`.
+
+Each attestation file must be a valid JSON object. You can use [Generate Build Provenance](https://github.com/buildkite-plugins/generate-build-provenance-buildkite-plugin) plugin to generate a valid build provenance attestation in your Buildkite pipeline.
+
+If `artifact_build_id` is specified, attestations will be downloaded from the relevant build artifact storage.
+
 #### `artifact_build_id` (string, optional)
 
 Configures the plugin to download artifacts from a different build, referenced by its UUID.
@@ -90,3 +126,25 @@ steps:
           registry: "acme-corp/awesome-logger"
           artifact_build_id: "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
 ```
+
+### Building and publishing with build provenance attestation
+
+```yaml
+steps:
+  - label: "Build Gem"
+    key: "build-gem"
+    command: "gem build awesome-logger.gemspec"
+    artifact_paths: "awesome-logger-*.gem" # upload to build artifact storage
+    plugins:
+      - generate-build-provenance#v3.0.0:
+          artifacts: "awesome-logger-*.gem" # publish from build artifact storage
+          attestation_name: "gem-build.attestation.json"
+
+  - label: "Publish Gem"
+    depends_on: "build-gem"
+    plugins:
+      - publish-to-packages#v2.0.0:
+          artifacts: "awesome-logger-*.gem" # publish from build artifact storage
+          registry: "acme-corp/awesome-logger"
+          attestations: "gem-build.attestation.json"
+```
diff --git a/hooks/command b/hooks/command
@@ -19,13 +19,13 @@ SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
 ARTIFACTS="$(plugin_read_config ARTIFACTS "")"
 REGISTRY="$(plugin_read_config REGISTRY "")"
 ARTIFACT_BUILD_ID="$(plugin_read_config ARTIFACT_BUILD_ID "")"
-PROVENANCE_BUNDLE="$(plugin_read_config PROVENANCE_BUNDLE "")"
+ATTESTATIONS="$(plugin_read_list ATTESTATIONS "")"
 
 TMP_DIR=$(mktemp -d)
 ARTIFACTS_DIR="${TMP_DIR}/artifacts"
-PROVENANCE_DIR="${TMP_DIR}/provenance"
+ATTESTATIONS_DIR="${TMP_DIR}/attestations"
 
-mkdir -p "${ARTIFACTS_DIR}" "${PROVENANCE_DIR}"
+mkdir -p "${ARTIFACTS_DIR}" "${ATTESTATIONS_DIR}"
 
 echo "~~~ 🚚 Download artifacts"
 
@@ -39,23 +39,19 @@ fi
 # shellcheck disable=SC2086
 buildkite-agent artifact download ${ARTIFACTS} ${ARTIFACTS_DIR} ${BUILD_OPTION}
 
-if [ "${PROVENANCE_BUNDLE}" != "" ]; then
-  # shellcheck disable=SC2086
-  buildkite-agent artifact download ${PROVENANCE_BUNDLE} ${PROVENANCE_DIR} ${BUILD_OPTION}
+if [ "${ATTESTATIONS}" != "" ]; then
+  for FILE in $ATTESTATIONS; do
+    # shellcheck disable=SC2086
+    buildkite-agent artifact download ${FILE} ${ATTESTATIONS_DIR} ${BUILD_OPTION}
+  done
 fi
 
 echo "+++ 🚚 Publishing to Packages"
 
-if [ "${PROVENANCE_BUNDLE}" != "" ]; then
-  PROVENANCE_BUNDLE_PATH="${PROVENANCE_DIR}/${PROVENANCE_BUNDLE}"
-else
-  PROVENANCE_BUNDLE_PATH=""
-fi
-
 python3 "${SCRIPT_DIR}/../main.py" \
   --artifacts-dir "${ARTIFACTS_DIR}" \
+  --attestations-dir "${ATTESTATIONS_DIR}" \
   --registry "${REGISTRY}" \
-  --provenance-bundle "${PROVENANCE_BUNDLE_PATH}" \
   --organization-slug "${BUILDKITE_ORGANIZATION_SLUG}"
 
 echo "~~~ 🚚 Clean up Publish to Packages"

diff --git a/lib/plugin.bash b/lib/plugin.bash
@@ -5,6 +5,29 @@ set -euo pipefail
 
 PLUGIN_PREFIX="PUBLISH_TO_PACKAGES"
 
+# Reads either a value or a list from the given env prefix
+function prefix_read_list() {
+  local prefix="$1"
+  local parameter="${prefix}_0"
+
+  if [ -n "${!parameter:-}" ]; then
+    local i=0
+    local parameter="${prefix}_${i}"
+    while [ -n "${!parameter:-}" ]; do
+      echo "${!parameter}"
+      i=$((i+1))
+      parameter="${prefix}_${i}"
+    done
+  elif [ -n "${!prefix:-}" ]; then
+    echo "${!prefix}"
+  fi
+}
+
+# Reads either a value or a list from plugin config
+function plugin_read_list() {
+  prefix_read_list "BUILDKITE_PLUGIN_${PLUGIN_PREFIX}_${1}"
+}
+
 # Reads a single value
 function plugin_read_config() {
   local var="BUILDKITE_PLUGIN_${PLUGIN_PREFIX}_${1}"

diff --git a/main.py b/main.py
@@ -1,28 +1,56 @@
 import os
 from glob import glob
+from pathlib import Path
 
 from package_publisher.cli_arguments import CliArguments
 from package_publisher.core import PackagePublisher
+from package_publisher.helpers import attestations_to_bundle
 
 arguments = CliArguments()
 
-publisher = PackagePublisher(registry=arguments.get_registry())
-
 artifacts_dir = arguments.get_artifacts_dir()
+if artifacts_dir == "":
+    print(
+        "Error: Missing --artifacts-dir argument. Example: --artifacts-dir ./artifacts"
+    )
+    exit(1)
+
 artifacts_glob = glob("{}/**/*".format(artifacts_dir), recursive=True)
-files = [path for path in artifacts_glob if os.path.isfile(path)]
+file_paths = [path for path in artifacts_glob if os.path.isfile(path)]
+
+attestations_dir = arguments.get_attestations_dir()
+
+if attestations_dir != "":
+    attestations_glob = glob("{}/**/*".format(attestations_dir), recursive=True)
+    attestation_files = [path for path in attestations_glob if os.path.isfile(path)]
+    attestation_bundle_path = attestations_to_bundle(attestation_files)
+else:
+    attestation_bundle_path = None
 
-for file in files:
+
+publisher = PackagePublisher(
+    registry=arguments.get_registry(),
+    attestation_bundle_path=attestation_bundle_path,
+)
+
+for file_path in file_paths:
     print(
         "Publishing {} → {}".format(
-            file.replace("{}/".format(artifacts_dir), ""), arguments.get_registry()
+            file_path.replace("{}/".format(artifacts_dir), ""), arguments.get_registry()
         )
     )
 
     response = publisher.upload_package(
-        file_path=file,
-        provenance_bundle_path=arguments.get_provenance_bundle(),
+        file_path=file_path,
     )
 
     print(" ✅ \033]1339;url={}\a".format(response["web_url"]))
     print("")
+
+
+if attestation_bundle_path is not None:
+    print("~~~ 🚚 Preview Attestation Bundle")
+    with open(attestation_bundle_path, "r", encoding="utf-8") as f:
+        print(f.read())
+
+    Path(attestation_bundle_path).unlink()
diff --git a/package_publisher/cli_arguments.py b/package_publisher/cli_arguments.py
@@ -10,7 +10,7 @@ def __init__(self) -> None:
         )
         parser.add_argument("--registry", default="")
         parser.add_argument("--artifacts-dir", default="")
-        parser.add_argument("--provenance-bundle", default="")
+        parser.add_argument("--attestations-dir", default="")
         parser.add_argument("--organization-slug", default="")
 
         self.arguments = parser.parse_args()
@@ -24,8 +24,8 @@ def get_registry(self) -> str:
     def get_artifacts_dir(self) -> str:
         return str(self.arguments.artifacts_dir).strip()
 
-    def get_provenance_bundle(self) -> str:
-        return str(self.arguments.provenance_bundle).strip()
+    def get_attestations_dir(self) -> str:
+        return str(self.arguments.attestations_dir).strip()
 
     def get_organization_slug(self) -> str:
         return str(self.arguments.organization_slug).strip()
diff --git a/package_publisher/core.py b/package_publisher/core.py
@@ -4,10 +4,13 @@
 
 
 class PackagePublisher:
-    def __init__(self, registry: str) -> None:
+    def __init__(
+        self, registry: str, attestation_bundle_path: str | None = None
+    ) -> None:
         (self.organization_slug, self.registry_slug) = registry.split("/")
+        self.attestation_bundle_path = attestation_bundle_path
 
-    def upload_package(self, file_path: str, provenance_bundle_path: str) -> Any:
+    def upload_package(self, file_path: str) -> Any:
         url = "https://api.buildkite.com/v2/packages/organizations/{}/registries/{}/packages".format(
             self.organization_slug, self.registry_slug
         )
@@ -20,8 +23,11 @@ def upload_package(self, file_path: str, provenance_bundle_path: str) -> Any:
         ]
         command += ["--form", "file=@{}".format(file_path)]
         command += (
-            ["--form", "provenance_bundle=@{}".format(provenance_bundle_path)]
-            if provenance_bundle_path != ""
+            [
+                "--form",
+                "attestation_bundle=@{}".format(self.attestation_bundle_path),
+            ]
+            if self.attestation_bundle_path is not None
             else []
         )
         command += [url]

diff --git a/package_publisher/helpers.py b/package_publisher/helpers.py
@@ -0,0 +1,24 @@
+import json
+
+from tempfile import NamedTemporaryFile
+
+
+def attestations_to_bundle(file_paths: list[str]) -> str | None:
+    if len(file_paths) <= 0:
+        return None
+
+    bundle_file = NamedTemporaryFile(delete=False)
+
+    for file_path in file_paths:
+        with open(file_path, "r", encoding="utf-8") as file:
+            try:
+                content = json.loads(file.read())
+            except json.decoder.JSONDecodeError as error:
+                print("Error parsing JSON in attestation: {}".format(file_path))
+                print("  {}".format(error))
+                exit(1)
+            bundle_file.write(bytearray(json.dumps(content), encoding="utf-8"))
+            bundle_file.write(bytearray("\n", encoding="utf-8"))
+
+    bundle_file.close()
+    return bundle_file.name
diff --git a/plugin.yml b/plugin.yml
@@ -10,6 +10,10 @@ configuration:
       type: string
     registry:
       type: string
+    attestations:
+      anyOf:
+        - type: string
+        - type: array
     artifact_build_id:
       type: string
   required:

diff --git a/tests/test_core.py b/tests/test_core.py
@@ -60,7 +60,7 @@ def mock_run(args: str, **_: List[Any]) -> Any:
 
         pp = PackagePublisher(registry="acme-corp/awesome-gem")
 
-        pp.upload_package(file_path="/example/file.gem", provenance_bundle_path="")
+        pp.upload_package(file_path="/example/file.gem")
 
         mock_subprocess.run.assert_called_with(
             [
@@ -79,7 +79,7 @@ def mock_run(args: str, **_: List[Any]) -> Any:
         )
 
     @patch("package_publisher.core.subprocess")
-    def test_upload_package_shells_out_correctly_with_provenance_path(
+    def test_upload_package_shells_out_correctly_when_attestation_bundle_path_is_set(
         self, mock_subprocess: Mock
     ) -> None:
         mock_subprocess.return_value = Mock(name="subprocess")
@@ -94,13 +94,13 @@ def mock_run(args: str, **_: List[Any]) -> Any:
 
         mock_subprocess.run.side_effect = mock_run
 
-        pp = PackagePublisher(registry="acme-corp/awesome-gem")
-
-        pp.upload_package(
-            file_path="/example/file.gem",
-            provenance_bundle_path="/example/provenance.json",
+        pp = PackagePublisher(
+            registry="acme-corp/awesome-gem",
+            attestation_bundle_path="/path/to/attestation_bundle.jsonl",
         )
 
+        pp.upload_package(file_path="/example/file.gem")
+
         mock_subprocess.run.assert_called_with(
             [
                 "curl",
@@ -111,7 +111,7 @@ def mock_run(args: str, **_: List[Any]) -> Any:
                 "--form",
                 "file=@/example/file.gem",
                 "--form",
-                "provenance_bundle=@/example/provenance.json",
+                "attestation_bundle=@/path/to/attestation_bundle.jsonl",
                 "https://api.buildkite.com/v2/packages/organizations/acme-corp/registries/awesome-gem/packages",
             ],
             capture_output=True,
@@ -141,8 +141,6 @@ def mock_run(args: str, **_: List[Any]) -> Any:
 
         pp = PackagePublisher(registry="acme-corp/awesome-gem")
 
-        result = pp.upload_package(
-            file_path="/example/file.gem", provenance_bundle_path=""
-        )
+        result = pp.upload_package(file_path="/example/file.gem")
 
         self.assertEqual(result, dict(field_1="value_1", field_2="value_2"))