From a48f58a60c728d07fad8359d95e2854385da8452 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Sun, 23 Jun 2024 17:39:17 -0700 Subject: [PATCH] tarball uses hermetic coreutils --- oci/private/tarball.bzl | 16 +++++++++++----- oci/private/tarball.sh.tpl | 21 +++++++++++---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/oci/private/tarball.bzl b/oci/private/tarball.bzl index f9a0c762..3bc4bcbc 100644 --- a/oci/private/tarball.bzl +++ b/oci/private/tarball.bzl @@ -71,9 +71,9 @@ attrs = { used to load the image into the local engine when using `bazel run` on this oci_tarball. By default, we look for `docker` or `podman` on the PATH, and run the `load` command. - + > Note that rules_docker has an "incremental loader" which is faster than oci_tarball by design. - > Something similar can be done for oci_tarball. + > Something similar can be done for oci_tarball. > See [loader.sh](/examples/incremental_loader/loader.sh) and explanation about [how](/examples/incremental_loader/README.md) it works. See the _run_template attribute for the script that calls this loader tool. @@ -87,9 +87,9 @@ attrs = { default = Label("//oci/private:tarball_run.sh.tpl"), doc = """ \ The template used to load the container when using `bazel run` on this oci_tarball. - + See the `loader` attribute to replace the tool which is called. - Please reference the default template to see available substitutions. + Please reference the default template to see available substitutions. """, allow_single_file = True, ), @@ -100,6 +100,7 @@ attrs = { def _tarball_impl(ctx): jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] + coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"] bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"] image = ctx.file.image @@ -113,6 +114,7 @@ def _tarball_impl(ctx): substitutions = { "{{format}}": ctx.attr.format, "{{jq_path}}": jq.jqinfo.bin.path, + "{{coreutils_path}}": coreutils.coreutils_info.bin.path, "{{tar}}": bsdtar.tarinfo.binary.path, "{{image_dir}}": image.path, "{{output}}": mtree_spec.path, @@ -138,7 +140,10 @@ def _tarball_impl(ctx): executable = util.maybe_wrap_launcher_for_windows(ctx, executable), inputs = mtree_inputs, outputs = mtree_outputs, - tools = [jq.jqinfo.bin], + tools = [ + jq.jqinfo.bin, + coreutils.coreutils_info.bin, + ], mnemonic = "OCITarballManifest", ) @@ -198,6 +203,7 @@ oci_tarball = rule( doc = doc, toolchains = [ "@bazel_tools//tools/sh:toolchain_type", + "@aspect_bazel_lib//lib:coreutils_toolchain_type", "@aspect_bazel_lib//lib:jq_toolchain_type", "@aspect_bazel_lib//lib:tar_toolchain_type", ], diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index 1eb6e42a..96a5d878 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -5,9 +5,10 @@ set -o pipefail -o errexit -o nounset readonly FORMAT="{{format}}" readonly JQ="{{jq_path}}" +readonly COREUTILS="{{coreutils_path}}" readonly TAR="{{tar}}" readonly IMAGE_DIR="{{image_dir}}" -readonly REPOTAGS=($(cat "{{tags}}")) +readonly REPOTAGS=($("${COREUTILS}" cat "{{tags}}")) readonly INDEX_FILE="${IMAGE_DIR}/index.json" readonly OUTPUT="{{output}}" @@ -17,14 +18,14 @@ readonly OUTPUT="{{output}}" function add_to_tar() { content=$1 tar_path=$2 - echo >>"${OUTPUT}" "${tar_path} uid=0 gid=0 mode=0755 time=1672560000 type=file content=${content}" + "${COREUTILS}" echo >>"${OUTPUT}" "${tar_path} uid=0 gid=0 mode=0755 time=1672560000 type=file content=${content}" } -MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') +MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | "${COREUTILS}" tr -d '"') MANIFESTS_LENGTH=$("${JQ}" -r '.manifests | length' "${INDEX_FILE}") if [[ "${MANIFESTS_LENGTH}" != 1 ]]; then - echo >&2 "Expected exactly one manifest in ${INDEX_FILE}" + "${COREUTILS}" echo >&2 "Expected exactly one manifest in ${INDEX_FILE}" exit 1 fi @@ -33,15 +34,15 @@ MEDIA_TYPE=$("${JQ}" -r ".manifests[0].mediaType" "${INDEX_FILE}") # Check that we know how to generate the output format given the input format. # We may expand the supported options here in the future, but for now, if [[ "${FORMAT}" != "docker" && "${FORMAT}" != "oci" ]]; then - echo >&2 "Unknown format: ${FORMAT}. Only support docker|oci" + "${COREUTILS}" echo >&2 "Unknown format: ${FORMAT}. Only support docker|oci" exit 1 fi if [[ "${FORMAT}" == "oci" && "${MEDIA_TYPE}" != "application/vnd.oci.image.index.v1+json" && "${MEDIA_TYPE}" != "application/vnd.docker.distribution.manifest.v2+json" ]]; then - echo >&2 "Format oci is only supported for oci_image_index targets but saw ${MEDIA_TYPE}" + "${COREUTILS}" echo >&2 "Format oci is only supported for oci_image_index targets but saw ${MEDIA_TYPE}" exit 1 fi if [[ "${FORMAT}" == "docker" && "${MEDIA_TYPE}" != "application/vnd.oci.image.manifest.v1+json" && "${MEDIA_TYPE}" != "application/vnd.docker.distribution.manifest.v2+json" ]]; then - echo >&2 "Format docker is only supported for oci_image targets but saw ${MEDIA_TYPE}" + "${COREUTILS}" echo >&2 "Format docker is only supported for oci_image targets but saw ${MEDIA_TYPE}" exit 1 fi @@ -51,7 +52,7 @@ if [[ "${FORMAT}" == "oci" ]]; then add_to_tar "${IMAGE_DIR}/oci-layout" oci-layout - INDEX_FILE_MANIFEST_DIGEST=$("${JQ}" -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') + INDEX_FILE_MANIFEST_DIGEST=$("${JQ}" -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | "${COREUTILS}" tr -d '"') INDEX_FILE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${INDEX_FILE_MANIFEST_DIGEST}" add_to_tar "${INDEX_FILE_MANIFEST_BLOB_PATH}" "blobs/${INDEX_FILE_MANIFEST_DIGEST}" @@ -83,7 +84,7 @@ if [[ "${FORMAT}" == "oci" ]]; then exit 0 fi -MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${IMAGE_DIR}/index.json" | tr -d '"') +MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${IMAGE_DIR}/index.json" | "${COREUTILS}" tr -d '"') MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}" CONFIG_DIGEST=$(${JQ} -r '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH}) @@ -93,7 +94,7 @@ LAYERS=$(${JQ} -cr '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" -for LAYER in $(${JQ} -r ".[]" <<< $LAYERS); do +for LAYER in $(${JQ} -r ".[]" <<< $LAYERS); do add_to_tar "${IMAGE_DIR}/blobs/${LAYER}" "blobs/${LAYER}.tar.gz" done