diff --git a/README.md b/README.md index 2760265..3056e5b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,37 @@ # Publish to Packages [![Build status](https://badge.buildkite.com/8dff045aea2a2227a4387e77941af1177230066dc459982c67.svg)](https://buildkite.com/buildkite/plugins-publish-to-packages) -A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that publishes [build artifacts](https://buildkite.com/docs/pipelines/artifacts) to [Buildkite Packages](https://buildkite.com/packages). +A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that publishes [build artifacts](https://buildkite.com/docs/pipelines/artifacts) and attestations to [Buildkite Packages](https://buildkite.com/packages). This plugin authenticates with Buildkite Packages using an [Agent OIDC token](https://buildkite.com/docs/agent/v3/cli-oidc), so your registry needs to be configured with a suitable [OIDC policy](https://buildkite.com/docs/packages/security/oidc#define-an-oidc-policy-for-a-registry). +## Quick Start + +### Minimal + +```yaml +steps: + - label: "Publish Gem" + plugins: + - publish-to-packages#v2.0.0: + artifacts: "awesome-logger-*.gem" + registry: "acme-corp/awesome-logger" +``` + +### The Works + +```yaml +steps: + - label: "Publish Gem" + plugins: + - publish-to-packages#v2.0.0: + artifacts: "awesome-logger-*.gem" + registry: "acme-corp/awesome-logger" + attestations: # optional + - "gem-build.attestation.json" + - "gem-package.attestation.json" + artifact_build_id: "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}" # optional +``` + ## Options #### `artifacts` (string, required) @@ -17,6 +45,14 @@ Buildkite Packages registry to publish to. - Full format is `/` (e.g. `acme-corp/awesome-logger`). - `` defaults to your Buildkite organization if omitted (e.g. `awesome-logger`). +#### `attestations` (string or list of strings, optional) + +One or more attestations from artifact storage to publish along with each package created from `artifacts`. + +Each attestation file must be a valid JSON object. You can use [Generate Build Provenance](https://github.com/buildkite-plugins/generate-build-provenance-buildkite-plugin) plugin to generate a valid build provenance attestation in your Buildkite pipeline. + +If `artifact_build_id` is specified, attestations will be downloaded from the relevant build artifact storage. + #### `artifact_build_id` (string, optional) Configures the plugin to download artifacts from a different build, referenced by its UUID. @@ -90,3 +126,25 @@ steps: registry: "acme-corp/awesome-logger" artifact_build_id: "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}" ``` + +### Building and publishing with build provenance attestation + +```yaml +steps: + - label: "Build Gem" + key: "build-gem" + command: "gem build awesome-logger.gemspec" + artifact_paths: "awesome-logger-*.gem" # upload to build artifact storage + plugins: + - generate-build-provenance#v3.0.0: + artifacts: "awesome-logger-*.gem" # publish from build artifact storage + attestation_name: "gem-build.attestation.json" + + - label: "Publish Gem" + depends_on: "build-gem" + plugins: + - publish-to-packages#v2.0.0: + artifacts: "awesome-logger-*.gem" # publish from build artifact storage + registry: "acme-corp/awesome-logger" + attestations: "gem-build.attestation.json" +``` diff --git a/hooks/command b/hooks/command index 0e53978..e8a7458 100755 --- a/hooks/command +++ b/hooks/command @@ -19,13 +19,13 @@ SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" ARTIFACTS="$(plugin_read_config ARTIFACTS "")" REGISTRY="$(plugin_read_config REGISTRY "")" ARTIFACT_BUILD_ID="$(plugin_read_config ARTIFACT_BUILD_ID "")" -PROVENANCE_BUNDLE="$(plugin_read_config PROVENANCE_BUNDLE "")" +ATTESTATIONS="$(plugin_read_list ATTESTATIONS "")" TMP_DIR=$(mktemp -d) ARTIFACTS_DIR="${TMP_DIR}/artifacts" -PROVENANCE_DIR="${TMP_DIR}/provenance" +ATTESTATIONS_DIR="${TMP_DIR}/attestations" -mkdir -p "${ARTIFACTS_DIR}" "${PROVENANCE_DIR}" +mkdir -p "${ARTIFACTS_DIR}" "${ATTESTATIONS_DIR}" echo "~~~ 🚚 Download artifacts" @@ -39,23 +39,19 @@ fi # shellcheck disable=SC2086 buildkite-agent artifact download ${ARTIFACTS} ${ARTIFACTS_DIR} ${BUILD_OPTION} -if [ "${PROVENANCE_BUNDLE}" != "" ]; then - # shellcheck disable=SC2086 - buildkite-agent artifact download ${PROVENANCE_BUNDLE} ${PROVENANCE_DIR} ${BUILD_OPTION} +if [ "${ATTESTATIONS}" != "" ]; then + for FILE in $ATTESTATIONS; do + # shellcheck disable=SC2086 + buildkite-agent artifact download ${FILE} ${ATTESTATIONS_DIR} ${BUILD_OPTION} + done fi echo "+++ 🚚 Publishing to Packages" -if [ "${PROVENANCE_BUNDLE}" != "" ]; then - PROVENANCE_BUNDLE_PATH="${PROVENANCE_DIR}/${PROVENANCE_BUNDLE}" -else - PROVENANCE_BUNDLE_PATH="" -fi - python3 "${SCRIPT_DIR}/../main.py" \ --artifacts-dir "${ARTIFACTS_DIR}" \ + --attestations-dir "${ATTESTATIONS_DIR}" \ --registry "${REGISTRY}" \ - --provenance-bundle "${PROVENANCE_BUNDLE_PATH}" \ --organization-slug "${BUILDKITE_ORGANIZATION_SLUG}" echo "~~~ 🚚 Clean up Publish to Packages" diff --git a/lib/plugin.bash b/lib/plugin.bash index f1ad336..60b13a8 100644 --- a/lib/plugin.bash +++ b/lib/plugin.bash @@ -5,6 +5,29 @@ set -euo pipefail PLUGIN_PREFIX="PUBLISH_TO_PACKAGES" +# Reads either a value or a list from the given env prefix +function prefix_read_list() { + local prefix="$1" + local parameter="${prefix}_0" + + if [ -n "${!parameter:-}" ]; then + local i=0 + local parameter="${prefix}_${i}" + while [ -n "${!parameter:-}" ]; do + echo "${!parameter}" + i=$((i+1)) + parameter="${prefix}_${i}" + done + elif [ -n "${!prefix:-}" ]; then + echo "${!prefix}" + fi +} + +# Reads either a value or a list from plugin config +function plugin_read_list() { + prefix_read_list "BUILDKITE_PLUGIN_${PLUGIN_PREFIX}_${1}" +} + # Reads a single value function plugin_read_config() { local var="BUILDKITE_PLUGIN_${PLUGIN_PREFIX}_${1}" diff --git a/main.py b/main.py index 20af76b..26632bf 100644 --- a/main.py +++ b/main.py @@ -1,28 +1,56 @@ import os from glob import glob +from pathlib import Path from package_publisher.cli_arguments import CliArguments from package_publisher.core import PackagePublisher +from package_publisher.helpers import attestations_to_bundle arguments = CliArguments() -publisher = PackagePublisher(registry=arguments.get_registry()) - artifacts_dir = arguments.get_artifacts_dir() +if artifacts_dir == "": + print( + "Error: Missing --artifacts-dir argument. Example: --artifacts-dir ./artifacts" + ) + exit(1) + artifacts_glob = glob("{}/**/*".format(artifacts_dir), recursive=True) -files = [path for path in artifacts_glob if os.path.isfile(path)] +file_paths = [path for path in artifacts_glob if os.path.isfile(path)] + +attestations_dir = arguments.get_attestations_dir() + +if attestations_dir != "": + attestations_glob = glob("{}/**/*".format(attestations_dir), recursive=True) + attestation_files = [path for path in attestations_glob if os.path.isfile(path)] + attestation_bundle_path = attestations_to_bundle(attestation_files) +else: + attestation_bundle_path = None -for file in files: + +publisher = PackagePublisher( + registry=arguments.get_registry(), + attestation_bundle_path=attestation_bundle_path, +) + +for file_path in file_paths: print( "Publishing {} → {}".format( - file.replace("{}/".format(artifacts_dir), ""), arguments.get_registry() + file_path.replace("{}/".format(artifacts_dir), ""), arguments.get_registry() ) ) response = publisher.upload_package( - file_path=file, - provenance_bundle_path=arguments.get_provenance_bundle(), + file_path=file_path, ) print(" ✅ \033]1339;url={}\a".format(response["web_url"])) print("") + + +if attestation_bundle_path is not None: + print("~~~ 🚚 Preview Attestation Bundle") + with open(attestation_bundle_path, "r", encoding="utf-8") as f: + print(f.read()) + + Path(attestation_bundle_path).unlink() diff --git a/package_publisher/cli_arguments.py b/package_publisher/cli_arguments.py index 3a677a5..3312216 100644 --- a/package_publisher/cli_arguments.py +++ b/package_publisher/cli_arguments.py @@ -10,7 +10,7 @@ def __init__(self) -> None: ) parser.add_argument("--registry", default="") parser.add_argument("--artifacts-dir", default="") - parser.add_argument("--provenance-bundle", default="") + parser.add_argument("--attestations-dir", default="") parser.add_argument("--organization-slug", default="") self.arguments = parser.parse_args() @@ -24,8 +24,8 @@ def get_registry(self) -> str: def get_artifacts_dir(self) -> str: return str(self.arguments.artifacts_dir).strip() - def get_provenance_bundle(self) -> str: - return str(self.arguments.provenance_bundle).strip() + def get_attestations_dir(self) -> str: + return str(self.arguments.attestations_dir).strip() def get_organization_slug(self) -> str: return str(self.arguments.organization_slug).strip() diff --git a/package_publisher/core.py b/package_publisher/core.py index 8a41df9..734e7b0 100644 --- a/package_publisher/core.py +++ b/package_publisher/core.py @@ -4,10 +4,13 @@ class PackagePublisher: - def __init__(self, registry: str) -> None: + def __init__( + self, registry: str, attestation_bundle_path: str | None = None + ) -> None: (self.organization_slug, self.registry_slug) = registry.split("/") + self.attestation_bundle_path = attestation_bundle_path - def upload_package(self, file_path: str, provenance_bundle_path: str) -> Any: + def upload_package(self, file_path: str) -> Any: url = "https://api.buildkite.com/v2/packages/organizations/{}/registries/{}/packages".format( self.organization_slug, self.registry_slug ) @@ -20,8 +23,11 @@ def upload_package(self, file_path: str, provenance_bundle_path: str) -> Any: ] command += ["--form", "file=@{}".format(file_path)] command += ( - ["--form", "provenance_bundle=@{}".format(provenance_bundle_path)] - if provenance_bundle_path != "" + [ + "--form", + "attestation_bundle=@{}".format(self.attestation_bundle_path), + ] + if self.attestation_bundle_path is not None else [] ) command += [url] diff --git a/package_publisher/helpers.py b/package_publisher/helpers.py new file mode 100644 index 0000000..d75f7e6 --- /dev/null +++ b/package_publisher/helpers.py @@ -0,0 +1,24 @@ +import json + +from tempfile import NamedTemporaryFile + + +def attestations_to_bundle(file_paths: list[str]) -> str | None: + if len(file_paths) <= 0: + return None + + bundle_file = NamedTemporaryFile(delete=False) + + for file_path in file_paths: + with open(file_path, "r", encoding="utf-8") as file: + try: + content = json.loads(file.read()) + except json.decoder.JSONDecodeError as error: + print("Error parsing JSON in attestation: {}".format(file_path)) + print(" {}".format(error)) + exit(1) + bundle_file.write(bytearray(json.dumps(content), encoding="utf-8")) + bundle_file.write(bytearray("\n", encoding="utf-8")) + + bundle_file.close() + return bundle_file.name diff --git a/plugin.yml b/plugin.yml index 1f67c6d..f0f1c9d 100644 --- a/plugin.yml +++ b/plugin.yml @@ -10,6 +10,10 @@ configuration: type: string registry: type: string + attestations: + anyOf: + - type: string + - type: array artifact_build_id: type: string required: diff --git a/tests/test_core.py b/tests/test_core.py index 72dc046..3f59c51 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -60,7 +60,7 @@ def mock_run(args: str, **_: List[Any]) -> Any: pp = PackagePublisher(registry="acme-corp/awesome-gem") - pp.upload_package(file_path="/example/file.gem", provenance_bundle_path="") + pp.upload_package(file_path="/example/file.gem") mock_subprocess.run.assert_called_with( [ @@ -79,7 +79,7 @@ def mock_run(args: str, **_: List[Any]) -> Any: ) @patch("package_publisher.core.subprocess") - def test_upload_package_shells_out_correctly_with_provenance_path( + def test_upload_package_shells_out_correctly_when_attestation_bundle_path_is_set( self, mock_subprocess: Mock ) -> None: mock_subprocess.return_value = Mock(name="subprocess") @@ -94,13 +94,13 @@ def mock_run(args: str, **_: List[Any]) -> Any: mock_subprocess.run.side_effect = mock_run - pp = PackagePublisher(registry="acme-corp/awesome-gem") - - pp.upload_package( - file_path="/example/file.gem", - provenance_bundle_path="/example/provenance.json", + pp = PackagePublisher( + registry="acme-corp/awesome-gem", + attestation_bundle_path="/path/to/attestation_bundle.jsonl", ) + pp.upload_package(file_path="/example/file.gem") + mock_subprocess.run.assert_called_with( [ "curl", @@ -111,7 +111,7 @@ def mock_run(args: str, **_: List[Any]) -> Any: "--form", "file=@/example/file.gem", "--form", - "provenance_bundle=@/example/provenance.json", + "attestation_bundle=@/path/to/attestation_bundle.jsonl", "https://api.buildkite.com/v2/packages/organizations/acme-corp/registries/awesome-gem/packages", ], capture_output=True, @@ -141,8 +141,6 @@ def mock_run(args: str, **_: List[Any]) -> Any: pp = PackagePublisher(registry="acme-corp/awesome-gem") - result = pp.upload_package( - file_path="/example/file.gem", provenance_bundle_path="" - ) + result = pp.upload_package(file_path="/example/file.gem") self.assertEqual(result, dict(field_1="value_1", field_2="value_2")) diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..e7f3365 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,88 @@ +# Run tests with: python3 -m unittest tests/*.py + +import json +import unittest +from tempfile import NamedTemporaryFile +from unittest.mock import Mock, patch + +from package_publisher.helpers import attestations_to_bundle + + +class AttestationsToBundleTests(unittest.TestCase): + def test_it_returns_none_if_file_paths_is_empty(self) -> None: + result = attestations_to_bundle([]) + self.assertIsNone(result) + + def test_it_bundles_up_single_valid_json_correctly(self) -> None: + payload = json.dumps( + dict(a=1, b="two", c=dict(d="buckle my", e="shoe")), indent=2 + ) + + attestation_file = NamedTemporaryFile(delete=False) + attestation_file.write(bytearray(payload, encoding="utf-8")) + attestation_file.close() + + bundle_file_path = str(attestations_to_bundle([attestation_file.name])) + + with open(bundle_file_path, "r", encoding="utf-8") as f: + result = f.read() + self.assertEqual( + result, '{"a": 1, "b": "two", "c": {"d": "buckle my", "e": "shoe"}}\n' + ) + + def test_it_bundles_up_multiple_valid_json_correctly(self) -> None: + payloads = [ + json.dumps(dict(a=1, b="two", c=dict(d="buckle my", e="shoe")), indent=2), + json.dumps( + dict(a=3, b="four", c=dict(d="knock at", e="the door")), indent=2 + ), + json.dumps(dict(a=5, b="six", c=dict(d="pick up", e="sticks")), indent=2), + ] + + def payloads_to_files(payload: str) -> str: + file = NamedTemporaryFile(delete=False) + file.write(bytearray(payload, encoding="utf-8")) + file.close() + return file.name + + filenames = list(map(payloads_to_files, payloads)) + + bundle_file_path = str(attestations_to_bundle(filenames)) + + with open(bundle_file_path, "r", encoding="utf-8") as f: + result = f.read() + self.assertEqual( + result, + '{"a": 1, "b": "two", "c": {"d": "buckle my", "e": "shoe"}}\n' + '{"a": 3, "b": "four", "c": {"d": "knock at", "e": "the door"}}\n' + '{"a": 5, "b": "six", "c": {"d": "pick up", "e": "sticks"}}\n', + ) + + @patch("builtins.print") # mock_print + @patch("builtins.exit") # mock_exit + def test_it_exits_if_json_is_invalid( + self, + mock_exit: Mock, + mock_print: Mock, # mainly to suppress print output in unittest + ) -> None: + + class ExitException(Exception): + pass + + # Pop the call stack when `exit` is called + def mock_exit_fn(_: int) -> None: + raise ExitException + + mock_exit.side_effect = mock_exit_fn + + payload = "INVALID JSON" + + attestation_file = NamedTemporaryFile(delete=False) + attestation_file.write(bytearray(payload, encoding="utf-8")) + attestation_file.close() + + with self.assertRaises(ExitException): + attestations_to_bundle([attestation_file.name]) + + mock_exit.assert_called_with(1) + self.assertEqual(len(mock_print.call_args_list), 2)