From d33a8a9c569c7621f00ffe274a67e7511b828625 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Mon, 23 Dec 2024 18:14:15 +0000 Subject: [PATCH 1/5] krane: statically link krane into twoliter This change implements a small FFI library in golang which emulates the `krane` program from google/go-containerregistry. Prior to this change, we'd experimented with two approaches to using `krane` within twoliter, but both approaches had unfortunate complications: * Writing `krane` to a sealed anonymous file and executing it -- this approach would cause twoliter to be killed by some threat detection software. Executing sealed anonymous files is commonly restricted on Linux systems * Writing `krane` to a tempfile and executing it -- it's difficult to get Rust to reliably delete named tempfiles, which led to twoliter leaking tempfiles of the `krane` binary. --- .github/workflows/rust.yml | 2 - Cargo.lock | 11 +- Cargo.toml | 6 +- tools/attribution/attribution.sh | 19 +- tools/krane/Cargo.toml | 8 +- tools/krane/README.md | 8 +- tools/krane/build.rs | 109 ++------ tools/krane/go-src/go.mod | 64 +++++ tools/krane/go-src/go.sum | 175 +++++++++++++ tools/krane/go-src/main.go | 108 ++++++++ tools/krane/hashes/crane | 2 - ...001-krane-update-ecr-login-to-v0.9.0.patch | 233 ------------------ tools/krane/src/lib.rs | 139 ++++++++--- tools/oci-cli-wrapper/Cargo.toml | 4 +- tools/oci-cli-wrapper/src/cli.rs | 86 ------- tools/oci-cli-wrapper/src/crane.rs | 145 ++++++++--- tools/oci-cli-wrapper/src/lib.rs | 52 ++-- tools/pubsys/src/kit/publish_kit/mod.rs | 2 +- twoliter/Cargo.toml | 2 +- twoliter/src/project/lock/mod.rs | 11 +- twoliter/src/tools.rs | 2 - 21 files changed, 613 insertions(+), 575 deletions(-) create mode 100644 tools/krane/go-src/go.mod create mode 100644 tools/krane/go-src/go.sum create mode 100644 tools/krane/go-src/main.go delete mode 100644 tools/krane/hashes/crane delete mode 100644 tools/krane/patches/0001-krane-update-ecr-login-to-v0.9.0.patch delete mode 100644 tools/oci-cli-wrapper/src/cli.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 181cb69f..4e2329d4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,6 +20,4 @@ jobs: - uses: actions/setup-go@v5 with: go-version: "^1.18" - # Install `patch`, needed to build `krane-bundle` - - run: sudo apt-get install -y patch - run: make build diff --git a/Cargo.lock b/Cargo.lock index 5231fc94..2039fada 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1896,14 +1896,11 @@ dependencies = [ ] [[package]] -name = "krane-bundle" +name = "krane-static" version = "0.1.0" dependencies = [ "anyhow", - "flate2", - "lazy_static", - "tar", - "tempfile", + "libc", "which", ] @@ -2165,7 +2162,7 @@ name = "oci-cli-wrapper" version = "0.1.0" dependencies = [ "async-trait", - "krane-bundle", + "krane-static", "log", "olpc-cjson", "regex", @@ -3872,7 +3869,7 @@ dependencies = [ "filetime", "flate2", "futures", - "krane-bundle", + "krane-static", "log", "oci-cli-wrapper", "olpc-cjson", diff --git a/Cargo.toml b/Cargo.toml index 7e1f8b9f..82a10165 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,8 +38,8 @@ targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] [workspace.metadata.cross.build] pre-build = [ - # install golang and patch for krane-bundle - "apt update && apt --assume-yes install golang-1.22 patch", + # install golang and patch for krane-static + "apt update && apt --assume-yes install golang-1.22", "update-alternatives --install /usr/bin/go go /usr/lib/go-1.22/bin/go 10", # give the builder access to the go build and module caches "mkdir /.cache && chmod a+rw /.cache", @@ -51,7 +51,7 @@ bottlerocket-types = { version = "0.0.14", git = "https://github.com/bottlerocke bottlerocket-variant = { version = "0.1", path = "tools/bottlerocket-variant" } buildsys = { version = "0.1", path = "tools/buildsys", lib = true, artifact = [ "bin:buildsys" ] } buildsys-config = { version = "0.1", path = "tools/buildsys-config" } -krane-bundle = { version = "0.1", path = "tools/krane" } +krane-static = { version = "0.1", path = "tools/krane" } oci-cli-wrapper = { version = "0.1", path = "tools/oci-cli-wrapper" } parse-datetime = { version = "0.1", path = "tools/parse-datetime" } pipesys = { version = "0.1", path = "tools/pipesys", lib = true, artifact = [ "bin:pipesys" ] } diff --git a/tools/attribution/attribution.sh b/tools/attribution/attribution.sh index 05de4a29..630965d7 100755 --- a/tools/attribution/attribution.sh +++ b/tools/attribution/attribution.sh @@ -26,23 +26,20 @@ echo "Clarifying crate dependency licenses..." cargo --locked Cargo.toml # =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= -# go-containerregistry -pushd /src/tools/krane -../build-cache-fetch hashes/crane -TARBALL=$(grep -oP '\(\K[^\)]*' hashes/crane) -GO_CONTAINERREGISTRY_UNPACK_DIR=$(mktemp -d) -tar --strip-components=1 -xvf "${TARBALL}" -C "${GO_CONTAINERREGISTRY_UNPACK_DIR}" - -pushd "${GO_CONTAINERREGISTRY_UNPACK_DIR}/cmd/krane" +# krane-static +echo "Clarifying golang dependencies of krane-static" +KRANE_STATIC_VENDOR_DIR=$(mktemp -d) +cp -r /src/tools/krane/go-src/* "${KRANE_STATIC_VENDOR_DIR}" + +pushd "${KRANE_STATIC_VENDOR_DIR}" go mod vendor popd /usr/libexec/tools/bottlerocket-license-scan \ --clarify /src/clarify.toml \ --spdx-data /usr/libexec/tools/spdx-data \ - --out-dir ${LICENSEDIR}/krane \ - go-vendor "${GO_CONTAINERREGISTRY_UNPACK_DIR}/cmd/krane/vendor" -popd + --out-dir ${LICENSEDIR}/krane-static \ + go-vendor "${KRANE_STATIC_VENDOR_DIR}/vendor" # =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= # cargo-make (we currently use cargo-make from the SDK, but will ship it in Twoliter in the future) diff --git a/tools/krane/Cargo.toml b/tools/krane/Cargo.toml index 74202a41..07f72698 100644 --- a/tools/krane/Cargo.toml +++ b/tools/krane/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "krane-bundle" +name = "krane-static" version = "0.1.0" authors = ["Sean P. Kelly "] license = "Apache-2.0 OR MIT" @@ -8,11 +8,7 @@ publish = false [dependencies] anyhow.workspace = true -flate2.workspace = true -lazy_static.workspace = true -tempfile.workspace = true +libc.workspace = true [build-dependencies] -flate2.workspace = true -tar.workspace = true which.workspace = true diff --git a/tools/krane/README.md b/tools/krane/README.md index 86572168..5fb73ebd 100644 --- a/tools/krane/README.md +++ b/tools/krane/README.md @@ -1,10 +1,8 @@ -## krane-bundle +## krane-static This crate packages the `krane` utility from [google/go-containerregistry]. -The utility is compiled by a build script, the output of which is compressed and stored in the Rust -crate as via `include_bytes!`. -At runtime, `krane-bundle` writes the decompressed binary to a temp file, passing the -filepath of that file to any caller. +The program is replicated as static library exposed via C FFI. +Rust bindings are provided which imitate `std::process::Command::output`. [google/go-containerregistry]: https://github.com/google/go-containerregistry diff --git a/tools/krane/build.rs b/tools/krane/build.rs index 1481e4c5..fbf54458 100644 --- a/tools/krane/build.rs +++ b/tools/krane/build.rs @@ -1,84 +1,38 @@ -use flate2::{read::GzDecoder, write::GzEncoder}; -use std::fs::File; -use std::io::{self, prelude::*}; -use std::path::{Path, PathBuf}; +use std::env; +use std::path::PathBuf; use std::process::Command; -use std::{env, fs}; -use tar::Archive; -const CRANE_VERSION: &str = "0.20.1"; - -const REQUIRED_TOOLS: &[&str] = &["patch", "go"]; +const REQUIRED_TOOLS: &[&str] = &["go"]; fn main() { let script_dir = env::current_dir().unwrap(); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - println!("cargo::rerun-if-changed=../build-cache-fetch"); - println!("cargo::rerun-if-changed=hashes/crane"); - println!("cargo::rerun-if-changed=patches"); + println!("cargo::rerun-if-changed=go-src"); ensure_required_tools_installed(); - // Download and checksum-verify crane - env::set_current_dir(&out_dir).expect("Failed to set current directory"); - Command::new(script_dir.join("../build-cache-fetch")) - .arg(script_dir.join("hashes/crane")) - .status() - .expect("Failed to execute build-cache-fetch"); - - // extract crane sources - let crane_archive = out_dir.join(format!("go-containerregistry-v{CRANE_VERSION}.tar.gz")); - let crane_tgz = File::open(&crane_archive).expect("Failed to open crane archive"); - let mut tar_archive = Archive::new(GzDecoder::new(crane_tgz)); - - let crane_output_dir = out_dir.join(format!("go-containerregistry-v{CRANE_VERSION}")); - tar_archive - .unpack(&crane_output_dir) - .expect("Failed to extract crane sources"); - - // Perform any local modifications - let crane_source_dir = crane_output_dir.join(format!("go-containerregistry-{CRANE_VERSION}")); - apply_source_patches(&crane_source_dir, script_dir.join("patches")); - - // build krane - let build_output_loc = out_dir.join("krane"); - Command::new("go") - .arg("build") + // build krane FFI wrapper + let build_output_loc = out_dir.join("libkrane.a"); + let exit_status = Command::new("go") .env("GOOS", get_goos()) .env("GOARCH", get_goarch()) + .arg("build") + .arg("-buildmode=c-archive") .arg("-o") .arg(&build_output_loc) - .current_dir(crane_source_dir.join("cmd/krane")) + .arg("main.go") + .current_dir(script_dir.join("go-src")) .status() .expect("Failed to build crane"); - // compress krane - let krane_gz_path = out_dir.join("krane.gz"); - let compressed_output_file = - File::create(&krane_gz_path).expect("Failed to crate krane.gz file"); - - let krane_binary = File::open(&build_output_loc).expect("Failed to open krane binary"); - let mut reader = io::BufReader::new(&krane_binary); - let mut encoder = GzEncoder::new(&compressed_output_file, flate2::Compression::best()); - - let mut buffer = Vec::with_capacity( - krane_binary - .metadata() - .expect("Failed to get krane binary metadata") - .len() as usize, + assert!( + exit_status.success(), + "Failed to build krane -- go compiler exited nonzero" ); - reader - .read_to_end(&mut buffer) - .expect("Failed to read krane binary"); - encoder - .write_all(&buffer) - .expect("Failed to write compressed krane binary"); - encoder - .finish() - .expect("Failed to finish writing compressed krane binary"); - println!("cargo::rustc-env=KRANE_GZ_PATH={}", krane_gz_path.display()); + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-link-lib=static=krane"); } fn ensure_required_tools_installed() { @@ -88,42 +42,12 @@ fn ensure_required_tools_installed() { } } -fn apply_source_patches(source_path: impl AsRef, patch_dir: impl AsRef) { - let source_path = source_path.as_ref(); - let patch_dir = patch_dir.as_ref(); - - let mut patches = fs::read_dir(patch_dir) - .expect("Failed to read patch directory") - .filter_map(|entry| entry.ok()) - .map(|entry| entry.path()) - .filter(|path| path.extension().map(|ext| ext == "patch").unwrap_or(false)) - .collect::>(); - patches.sort(); - - for patch in patches { - println!("Executing `patch -p1 -i '{}'`", patch.display()); - - let patch_status = Command::new("patch") - .current_dir(source_path) - .arg("-p1") - .arg("-i") - .arg(patch.as_os_str()) - .status() - .expect("Failed to execute patch command"); - - if !patch_status.success() { - panic!("Failed to apply patch '{}'", patch.display()); - } - } -} - fn get_goos() -> &'static str { let target_os = env::var("CARGO_CFG_TARGET_OS").expect("Failed to read CARGO_CFG_TARGET_OS"); match target_os.as_str() { "linux" => "linux", "windows" => "windows", "macos" => "darwin", - // Add more mappings as needed other => panic!("Unsupported target OS: {}", other), } } @@ -137,7 +61,6 @@ fn get_goarch() -> &'static str { "aarch64" => "arm64", "arm" => "arm", "wasm32" => "wasm", - // Add more mappings as needed other => panic!("Unsupported target architecture: {}", other), } } diff --git a/tools/krane/go-src/go.mod b/tools/krane/go-src/go.mod new file mode 100644 index 00000000..ae5b1a85 --- /dev/null +++ b/tools/krane/go-src/go.mod @@ -0,0 +1,64 @@ +module github.com/bottlerocket-os/twoliter/tools/krane/go-src + +go 1.22 + +require ( + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240910181921-bef5bd9384b7 + github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 + github.com/google/go-containerregistry v0.20.2 +) + +require ( + cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go v46.4.0+incompatible // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.28 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.30.5 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.33 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.32.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect + github.com/aws/smithy-go v1.20.4 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/docker/cli v27.1.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker-credential-helpers v0.8.2 // indirect + github.com/golang-jwt/jwt/v4 v4.2.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.15.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.30.0 // indirect +) diff --git a/tools/krane/go-src/go.sum b/tools/krane/go-src/go.sum new file mode 100644 index 00000000..a8814d1e --- /dev/null +++ b/tools/krane/go-src/go.sum @@ -0,0 +1,175 @@ +cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/Azure/azure-sdk-for-go v46.4.0+incompatible h1:fCN6Pi+tEiEwFa8RSmtVlFHRXEZ+DJm9gfx/MKqYWw4= +github.com/Azure/azure-sdk-for-go v46.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= +github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU= +github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/ecr v1.32.4 h1:nQAU2Yr+afkAvIV39mg7LrNYFNQP7ShwbmiJqx2fUKA= +github.com/aws/aws-sdk-go-v2/service/ecr v1.32.4/go.mod h1:keOS9j4fv5ASh7dV29lIpGw2QgoJwGFAyMU0uPvfax4= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.6 h1:D9C5XIIciGM6mRZTi7zDdFsBsPsgzbsPwwN0wLCymnc= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.6/go.mod h1:Mrlicf7xXyuelm+q8XVMblDxJq2pKpKGXiWx/3uqjqs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240910181921-bef5bd9384b7 h1:wPoMm1zbH1k1D2+sYx19AYMnmBOGPoJB5A022oTwURs= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240910181921-bef5bd9384b7/go.mod h1:skPUtbWfzU/dVg+S37iSI415qioEEeAR6Ez72cOHR+M= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= +github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= +github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/tools/krane/go-src/main.go b/tools/krane/go-src/main.go new file mode 100644 index 00000000..423930ca --- /dev/null +++ b/tools/krane/go-src/main.go @@ -0,0 +1,108 @@ +package main + +// This module provides simple C FFI bindings over the behavior of the `krane` binary from +// [google/go-containerregistry]. +// +// Arguments are provided as a C command line would be: an array of nul-terminated C strings +// alongside an integer representing the length of the array (argc/argv). +// +// The `krane` function also accepts two **C.char pointers which it will point to nul-terminated +// C strings representing the stdout and stderr of the called command. +// +// Note: These strings must be freed by the caller. +// +// [google/go-containerregistry]: https://github.com/google/go-containerregistry + +import ( + "io" + "os" + + "C" + "unsafe" + + ecr "github.com/awslabs/amazon-ecr-credential-helper/ecr-login" + "github.com/chrismellard/docker-credential-acr-env/pkg/credhelper" + "github.com/google/go-containerregistry/cmd/crane/cmd" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/authn/github" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/v1/google" +) +import ( + "bytes" +) + +func init() { + logs.Warn.SetOutput(os.Stderr) + logs.Progress.SetOutput(os.Stderr) +} + +//export krane +func krane(argc C.int, argv **C.char, stdout **C.char, stderr **C.char) C.int { + args := parseCArgs(argc, argv) + + var outBuffer, errBuffer bytes.Buffer + + statusCode := kraneMain(args, false, &outBuffer, &errBuffer) + + *stdout = C.CString(outBuffer.String()) + *stderr = C.CString(errBuffer.String()) + + return C.int(statusCode) +} + +//export krane_inherited_io +func krane_inherited_io(argc C.int, argv **C.char) C.int { + args := parseCArgs(argc, argv) + + statusCode := kraneMain(args, true, nil, nil) + + return C.int(statusCode) +} + +func parseCArgs(argc C.int, argv **C.char) []string { + args := make([]string, 0, argc) + for i := 0; i < int(argc); i++ { + cStr := C.GoString(*argv) + args = append(args, cStr) + argv = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(argv)) + unsafe.Sizeof(*argv))) + } + return args +} + +const ( + use = "krane" + short = "krane is a tool for managing container images" +) + +var ( + amazonKeychain authn.Keychain = authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(io.Discard))) + azureKeychain authn.Keychain = authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper()) +) + +func kraneMain(args []string, inherited bool, outBuffer *bytes.Buffer, errBuffer *bytes.Buffer) uint { + keychain := authn.NewMultiKeychain( + authn.DefaultKeychain, + google.Keychain, + github.Keychain, + amazonKeychain, + azureKeychain, + ) + + // Same as crane, but override usage and keychain. + root := cmd.New(use, short, []crane.Option{crane.WithAuthFromKeychain(keychain)}) + root.SetArgs(args) + if !inherited { + root.SetOut(outBuffer) + root.SetErr(errBuffer) + } + + if err := root.Execute(); err != nil { + return 1 + } else { + return 0 + } +} + +func main() {} diff --git a/tools/krane/hashes/crane b/tools/krane/hashes/crane deleted file mode 100644 index a0f35732..00000000 --- a/tools/krane/hashes/crane +++ /dev/null @@ -1,2 +0,0 @@ -# https://github.com/google/go-containerregistry/archive/refs/tags/v0.20.1.tar.gz -SHA512 (go-containerregistry-v0.20.1.tar.gz) = c323c5b78c35fb7af67641fa4ef1802b944f8bd908163ff40a952a0c190e2dd210100efba1fbc2064495cff28a60d5bc7ee98e510ec116522c7897539b02fad8 diff --git a/tools/krane/patches/0001-krane-update-ecr-login-to-v0.9.0.patch b/tools/krane/patches/0001-krane-update-ecr-login-to-v0.9.0.patch deleted file mode 100644 index 57281442..00000000 --- a/tools/krane/patches/0001-krane-update-ecr-login-to-v0.9.0.patch +++ /dev/null @@ -1,233 +0,0 @@ -From 179d47cd4a413e8da0f0c963f1d5cdc06704470b Mon Sep 17 00:00:00 2001 -From: "Sean P. Kelly" -Date: Tue, 3 Dec 2024 20:30:07 +0000 -Subject: [PATCH] krane: update ecr-login to v0.9.0 - ---- - cmd/krane/go.mod | 39 +++++++++++++----------- - cmd/krane/go.sum | 79 ++++++++++++++++++++++++------------------------ - 2 files changed, 61 insertions(+), 57 deletions(-) - -diff --git a/cmd/krane/go.mod b/cmd/krane/go.mod -index 3cc202b7..cf9c0204 100644 ---- a/cmd/krane/go.mod -+++ b/cmd/krane/go.mod -@@ -1,11 +1,13 @@ - module github.com/google/go-containerregistry/cmd/krane - --go 1.18 -+go 1.21 -+ -+toolchain go1.23.2 - - replace github.com/google/go-containerregistry => ../../ - - require ( -- github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230510185313-f5e39e5f34c7 -+ github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240910181921-bef5bd9384b7 - github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 - github.com/google/go-containerregistry v0.15.2 - ) -@@ -22,26 +24,27 @@ require ( - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect -- github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect -- github.com/aws/aws-sdk-go-v2/config v1.18.25 // indirect -- github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect -- github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect -- github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect -- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect -- github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect -- github.com/aws/aws-sdk-go-v2/service/ecr v1.18.11 // indirect -- github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.16.2 // indirect -- github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect -- github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect -- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect -- github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect -- github.com/aws/smithy-go v1.13.5 // indirect -+ github.com/aws/aws-sdk-go-v2 v1.30.5 // indirect -+ github.com/aws/aws-sdk-go-v2/config v1.27.33 // indirect -+ github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect -+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect -+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect -+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect -+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect -+ github.com/aws/aws-sdk-go-v2/service/ecr v1.32.4 // indirect -+ github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.6 // indirect -+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect -+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect -+ github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect -+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect -+ github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect -+ github.com/aws/smithy-go v1.20.4 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect - github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/cli v24.0.0+incompatible // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.0+incompatible // indirect -- github.com/docker/docker-credential-helpers v0.7.0 // indirect -+ github.com/docker/docker-credential-helpers v0.8.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect -@@ -52,7 +55,7 @@ require ( - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc3 // indirect - github.com/pkg/errors v0.9.1 // indirect -- github.com/sirupsen/logrus v1.9.1 // indirect -+ github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/vbatts/tar-split v0.11.3 // indirect -diff --git a/cmd/krane/go.sum b/cmd/krane/go.sum -index 45188955..a72be078 100644 ---- a/cmd/krane/go.sum -+++ b/cmd/krane/go.sum -@@ -28,42 +28,43 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ - github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= - github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= - github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= --github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= --github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= --github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q= --github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= --github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vqVZ+zLfdOa/UPFDp0= --github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= --github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w= --github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= --github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= --github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= --github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= --github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= --github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8= --github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= --github.com/aws/aws-sdk-go-v2/service/ecr v1.18.11 h1:wlTgmb/sCmVRJrN5De3CiHj4v/bTCgL5+qpdEd0CPtw= --github.com/aws/aws-sdk-go-v2/service/ecr v1.18.11/go.mod h1:Ce1q2jlNm8BVpjLaOnwnm5v2RClAbK6txwPljFzyW6c= --github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.16.2 h1:yflJrGmi1pXtP9lOpOeaNZyc0vXnJTuP2sor3nJcGGo= --github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.16.2/go.mod h1:uHtRE7aqXNmpeYL+7Ec7LacH5zC9+w2T5MBOeEKDdu0= --github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo= --github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= --github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY= --github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= --github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 h1:PkHIIJs8qvq0e5QybnZoG1K/9QTrLr9OsqCIo59jOBA= --github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= --github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 h1:2DQLAKDteoEDI8zpCzqBMaZlJuoE9iTYD0gFmXVax9E= --github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= --github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= --github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= --github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230510185313-f5e39e5f34c7 h1:G5IT+PEpFY0CDb3oITDP9tkmLrHkVD8Ny+elUmBqVYI= --github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230510185313-f5e39e5f34c7/go.mod h1:VVALgT1UESBh91dY0GprHnT1Z7mKd96VDk8qVy+bmu0= -+github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= -+github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= -+github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU= -+github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks= -+github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I= -+github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I= -+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= -+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= -+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= -+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= -+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= -+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= -+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -+github.com/aws/aws-sdk-go-v2/service/ecr v1.32.4 h1:nQAU2Yr+afkAvIV39mg7LrNYFNQP7ShwbmiJqx2fUKA= -+github.com/aws/aws-sdk-go-v2/service/ecr v1.32.4/go.mod h1:keOS9j4fv5ASh7dV29lIpGw2QgoJwGFAyMU0uPvfax4= -+github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.6 h1:D9C5XIIciGM6mRZTi7zDdFsBsPsgzbsPwwN0wLCymnc= -+github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.6/go.mod h1:Mrlicf7xXyuelm+q8XVMblDxJq2pKpKGXiWx/3uqjqs= -+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= -+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= -+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= -+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= -+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= -+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= -+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4= -+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= -+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE= -+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= -+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= -+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -+github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240910181921-bef5bd9384b7 h1:wPoMm1zbH1k1D2+sYx19AYMnmBOGPoJB5A022oTwURs= -+github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240910181921-bef5bd9384b7/go.mod h1:skPUtbWfzU/dVg+S37iSI415qioEEeAR6Ez72cOHR+M= - github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= - github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= - github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= - github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= - github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= --github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= - github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= - github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= - github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -@@ -75,8 +76,8 @@ github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m3 - github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= - github.com/docker/docker v24.0.0+incompatible h1:z4bf8HvONXX9Tde5lGBMQ7yCJgNahmJumdrStZAbeY4= - github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= --github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= --github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -+github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= -+github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= - github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= - github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= - github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -@@ -86,7 +87,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS - github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= - github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= - github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= --github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= - github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= - github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= - github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -@@ -109,8 +109,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb - github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= - github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= - github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= --github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= --github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= - github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= - github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= - github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -@@ -123,8 +123,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ - github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= - github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= - github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= --github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= - github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= - github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= - github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= - github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= -@@ -142,6 +143,7 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq - golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= - golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= - golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= - golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= - golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -@@ -166,11 +168,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w - golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= - golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= - golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= --golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= --golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -@@ -189,6 +189,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn - golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= - golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= - golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= - golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= --- -2.40.1 - diff --git a/tools/krane/src/lib.rs b/tools/krane/src/lib.rs index 2ffd3503..554ecfe3 100644 --- a/tools/krane/src/lib.rs +++ b/tools/krane/src/lib.rs @@ -1,60 +1,129 @@ -use anyhow::Result; -use flate2::read::GzDecoder; -use std::fs::{File, Permissions}; -use std::os::unix::fs::PermissionsExt; -use std::path::PathBuf; +//! Rust interface to the golang FFI bindings for krane. +use anyhow::{ensure, Context, Result}; +use std::ffi::{c_char, CStr, CString}; +use std::os::unix::process::ExitStatusExt; +use std::process::ExitStatus; +use std::ptr; -use tempfile::TempDir; +pub type KraneError = anyhow::Error; -const COMPRESSED_KRANE_BIN: &[u8] = include_bytes!(env!("KRANE_GZ_PATH")); +/// Calls the go-containerregistry `krane` binary. +/// +/// The function is a thin wrapper around a statically compiled version of the binary which is +/// called via the C FFI. +pub fn call_krane(args: &[impl AsRef]) -> Result { + let argv_owned = c_args(args)?; -lazy_static::lazy_static! { - pub static ref KRANE: Krane = Krane::seal().unwrap(); -} + let mut argv: Vec<*mut c_char> = argv_owned + .iter() + .map(|arg| arg.as_ptr() as *mut c_char) + .collect(); + let argc = argv.len() as i32; + + // stdout/stderr are written to buffers, to which these will eventually point + let mut c_stdout: *mut c_char = ptr::null_mut(); + let mut c_stderr: *mut c_char = ptr::null_mut(); + + let status_code = + unsafe { extern_krane::krane(argc, argv.as_mut_ptr(), &mut c_stdout, &mut c_stderr) }; + + let c_stdout = CStringBuffer(c_stdout); + let c_stderr = CStringBuffer(c_stderr); + + let stdout = c_stdout + .try_into() + .context("krane ffi returned null pointer for stdout")?; + + let stderr = c_stderr + .try_into() + .context("krane ffi returned null pointer for stderr")?; -#[derive(Debug)] -pub struct Krane { - // Hold the file in memory to keep the fd open - _tmp_dir: TempDir, - path: PathBuf, + Ok(std::process::Output { + stdout, + stderr, + status: ExitStatus::from_raw(status_code), + }) } -impl Krane { - fn seal() -> Result { - let tmp_dir = TempDir::new()?; - let path = tmp_dir.path().join("krane"); +/// Calls the go-containerregistry `krane` binary. +/// +/// Unlike `call_krane`, output goes directly to stdout/stderr. +pub fn call_krane_inherited_io(args: &[impl AsRef]) -> Result { + let argv_owned = c_args(args)?; - let mut krane_file = File::create(&path)?; - let permissions = Permissions::from_mode(0o755); - krane_file.set_permissions(permissions)?; + let mut argv: Vec<*mut c_char> = argv_owned + .iter() + .map(|arg| arg.as_ptr() as *mut c_char) + .collect(); + let argc = argv.len() as i32; - let mut krane_reader = GzDecoder::new(COMPRESSED_KRANE_BIN); + let status_code = unsafe { extern_krane::krane_inherited_io(argc, argv.as_mut_ptr()) }; - std::io::copy(&mut krane_reader, &mut krane_file)?; + Ok(ExitStatus::from_raw(status_code)) +} - Ok(Krane { - _tmp_dir: tmp_dir, - path, +fn c_args(args: &[impl AsRef]) -> Result> { + args.iter() + .map(|arg| { + CString::new(arg.as_ref().as_bytes()) + .map_err(|_| anyhow::Error::msg("krane args contained illegal null byte")) }) + .collect::>>() +} + +/// Wrapper type around null-terminated C Strings representing arbitrary byte buffers. +/// +/// frees the wrapped pointer on drop. +struct CStringBuffer(*mut c_char); + +impl Drop for CStringBuffer { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { libc::free(self.0 as *mut std::ffi::c_void) }; + } } +} + +impl TryInto> for CStringBuffer { + type Error = anyhow::Error; + + fn try_into(self) -> Result> { + ensure!(!self.0.is_null(), "FFI returned null pointer"); + Ok(unsafe { CStr::from_ptr(self.0).to_bytes().to_vec() }) + } +} + +mod extern_krane { + use std::os::raw::{c_char, c_int}; + + extern "C" { + pub(crate) fn krane( + argc: c_int, + argv: *mut *mut c_char, + stdout: *mut *mut c_char, + stderr: *mut *mut c_char, + ) -> c_int; + + pub(crate) fn krane_inherited_io(argc: c_int, argv: *mut *mut c_char) -> c_int; - pub fn path(&self) -> &PathBuf { - &self.path } } #[cfg(test)] mod test { use super::*; - use std::process::Command; #[test] fn test_krane_runs() { - let status = Command::new(KRANE.path()) - .arg("--help") - .output() - .expect("failed to run krane"); + let krane_res = call_krane(&["--help"]).unwrap(); + println!("{:?}", krane_res); - assert_eq!(status.status.code().unwrap(), 0); + assert!(krane_res.status.success()); + assert!(String::from_utf8_lossy(&krane_res.stdout).starts_with("krane")); + } + + #[test] + fn test_krane_inherited_io_doesnt_explode() { + call_krane_inherited_io(&["--help"]).unwrap(); } } diff --git a/tools/oci-cli-wrapper/Cargo.toml b/tools/oci-cli-wrapper/Cargo.toml index 9f1d56f4..6056047e 100644 --- a/tools/oci-cli-wrapper/Cargo.toml +++ b/tools/oci-cli-wrapper/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] async-trait.workspace = true -krane-bundle.workspace = true +krane-static.workspace = true log.workspace = true olpc-cjson.workspace = true regex.workspace = true @@ -17,5 +17,5 @@ serde_json.workspace = true snafu.workspace = true tar.workspace = true tempfile.workspace = true -tokio = { workspace = true, features = ["process"] } +tokio = { workspace = true, features = ["process", "rt-multi-thread"] } which.workspace = true diff --git a/tools/oci-cli-wrapper/src/cli.rs b/tools/oci-cli-wrapper/src/cli.rs deleted file mode 100644 index 8f7d86f9..00000000 --- a/tools/oci-cli-wrapper/src/cli.rs +++ /dev/null @@ -1,86 +0,0 @@ -use snafu::{ensure, ResultExt}; -use std::path::PathBuf; -use tokio::process::Command; - -use crate::{error, Result}; - -#[derive(Debug)] -pub(crate) struct CommandLine { - pub(crate) path: PathBuf, -} - -impl CommandLine { - pub(crate) async fn output(&self, args: &[&str], error_msg: String) -> Result> { - let debug_cmd = [ - vec![format!("{}", self.path.display())], - args.iter() - .map(|arg| format!("'{}'", arg)) - .collect::>(), - ] - .concat() - .join(", "); - - log::debug!("Executing [{debug_cmd}]",); - let output = Command::new(&self.path) - .args(args) - .output() - .await - .context(error::CommandFailedSnafu { message: error_msg })?; - - ensure!( - output.status.success(), - error::OperationFailedSnafu { - message: format!( - "[{debug_cmd}]: status: {} stderr: {} stdout: {}", - &output.status, - String::from_utf8_lossy(&output.stderr), - String::from_utf8_lossy(&output.stdout) - ), - program: self.path.clone(), - args: args.iter().map(|x| x.to_string()).collect::>() - } - ); - - log::debug!( - "[{debug_cmd}] stdout: {}", - String::from_utf8_lossy(&output.stdout).to_string() - ); - log::debug!( - "[{debug_cmd}] stderr: {}", - String::from_utf8_lossy(&output.stderr).to_string() - ); - - Ok(output.stdout) - } - - pub(crate) async fn spawn(&self, args: &[&str], error_msg: String) -> Result<()> { - log::debug!( - "Executing '{}' with args [{}]", - self.path.display(), - args.iter() - .map(|arg| format!("'{}'", arg)) - .collect::>() - .join(", ") - ); - let status = Command::new(&self.path) - .args(args) - .spawn() - .context(error::CommandFailedSnafu { - message: error_msg.clone(), - })? - .wait() - .await - .context(error::CommandFailedSnafu { - message: error_msg.clone(), - })?; - ensure!( - status.success(), - error::OperationFailedSnafu { - message: error_msg.clone(), - program: self.path.clone(), - args: args.iter().map(|x| x.to_string()).collect::>() - } - ); - Ok(()) - } -} diff --git a/tools/oci-cli-wrapper/src/crane.rs b/tools/oci-cli-wrapper/src/crane.rs index 1841ac38..37f4e57e 100644 --- a/tools/oci-cli-wrapper/src/crane.rs +++ b/tools/oci-cli-wrapper/src/crane.rs @@ -1,19 +1,17 @@ +use std::fmt::Debug; use std::fs::File; use std::path::Path; use async_trait::async_trait; -use snafu::ResultExt; +use krane_static::{call_krane, call_krane_inherited_io}; +use snafu::{ensure, ResultExt}; use tar::Archive as TarArchive; use tempfile::TempDir; -use crate::{ - cli::CommandLine, error, ConfigView, DockerArchitecture, ImageToolImpl, ImageView, Result, -}; +use crate::{error, ConfigView, DockerArchitecture, ImageToolImpl, ImageView, Result}; #[derive(Debug)] -pub struct CraneCLI { - pub(crate) cli: CommandLine, -} +pub struct CraneCLI; impl CraneCLI { /// Enables verbose logging of crane if debug logging is enabled. @@ -24,38 +22,107 @@ impl CraneCLI { cmd.into() } } + + fn debug_cmd(args: &[&str]) -> String { + [ + vec!["krane".to_string()], + args.iter() + .map(|arg| format!("'{}'", arg)) + .collect::>(), + ] + .concat() + .join(", ") + } + + /// Calls `krane` with the given arguments. + /// + /// Returns stdout if the process successfully completes. + async fn output(cmd: &[&str], error_msg: &str) -> Result> { + let args = Self::crane_cmd(cmd); + + log::debug!("Executing [{}]", Self::debug_cmd(cmd)); + + let fork_args = args.iter().map(|s| s.to_string()).collect::>(); + let output = tokio::task::spawn_blocking(move || call_krane(&fork_args)) + .await + .context(error::ForkSnafu)? + .context(error::CraneFFISnafu)?; + + log::debug!( + "[{}] stdout: {}", + Self::debug_cmd(&args), + String::from_utf8_lossy(&output.stdout).to_string() + ); + log::debug!( + "[{}] stderr: {}", + Self::debug_cmd(&args), + String::from_utf8_lossy(&output.stderr).to_string() + ); + + ensure!( + output.status.success(), + error::OperationFailedSnafu { + message: error_msg, + program: "krane", + args: args.iter().map(|x| x.to_string()).collect::>() + } + ); + + Ok(output.stdout) + } + + /// Calls `krane` with the given arguments. + /// + /// stdout/stderr is inherited from the current process. + async fn call(cmd: &[&str], error_msg: &str) -> Result<()> { + let args = Self::crane_cmd(cmd); + + log::debug!("Executing [{}]", Self::debug_cmd(cmd)); + + let fork_args = args.iter().map(|s| s.to_string()).collect::>(); + let status = tokio::task::spawn_blocking(move || call_krane_inherited_io(&fork_args)) + .await + .context(error::ForkSnafu)? + .context(error::CraneFFISnafu)?; + + ensure!( + status.success(), + error::OperationFailedSnafu { + message: error_msg, + program: "krane", + args: args.iter().map(|x| x.to_string()).collect::>() + } + ); + + Ok(()) + } } #[async_trait] impl ImageToolImpl for CraneCLI { async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> { let archive_path = path.to_string_lossy(); - self.cli - .spawn( - &Self::crane_cmd(&["pull", "--format", "oci", uri, archive_path.as_ref()]), - format!("failed to pull image archive from {}", uri), - ) - .await?; - Ok(()) + Self::call( + &["pull", "--format", "oci", uri, archive_path.as_ref()], + &format!("failed to pull image archive from {}", uri), + ) + .await } async fn get_manifest(&self, uri: &str) -> Result> { - self.cli - .output( - &Self::crane_cmd(&["manifest", uri]), - format!("failed to fetch manifest for resource at {}", uri), - ) - .await + Self::output( + &["manifest", uri], + &format!("failed to fetch manifest for resource at {}", uri), + ) + .await } async fn get_config(&self, uri: &str) -> Result { - let bytes = self - .cli - .output( - &Self::crane_cmd(&["config", uri]), - format!("failed to fetch image config from {}", uri), - ) - .await?; + let bytes = Self::output( + &["config", uri], + &format!("failed to fetch image config from {}", uri), + ) + .await?; let image_view: ImageView = serde_json::from_slice(bytes.as_slice()).context(error::ConfigDeserializeSnafu)?; Ok(image_view.config) @@ -70,12 +137,11 @@ impl ImageToolImpl for CraneCLI { oci_archive .unpack(temp_dir.path()) .context(error::ArchiveExtractSnafu)?; - self.cli - .spawn( - &Self::crane_cmd(&["push", &temp_dir.path().to_string_lossy(), uri]), - format!("failed to push image {}", uri), - ) - .await + Self::call( + &["push", &temp_dir.path().to_string_lossy(), uri], + &format!("failed to push image {}", uri), + ) + .await } async fn push_multi_platform_manifest( @@ -93,13 +159,10 @@ impl ImageToolImpl for CraneCLI { manifest_create_args.extend_from_slice(&["-m", image]) } manifest_create_args.extend_from_slice(&["-t", uri]); - self.cli - .output( - &Self::crane_cmd(&manifest_create_args), - format!("could not push multi-platform manifest to {}", uri), - ) - .await?; - - Ok(()) + Self::call( + &manifest_create_args, + &format!("could not push multi-platform manifest to {}", uri), + ) + .await } } diff --git a/tools/oci-cli-wrapper/src/lib.rs b/tools/oci-cli-wrapper/src/lib.rs index 1949543f..8accf8b3 100644 --- a/tools/oci-cli-wrapper/src/lib.rs +++ b/tools/oci-cli-wrapper/src/lib.rs @@ -12,36 +12,29 @@ //! metadata. In addition, in order to operate with OCI image format, the containerd-snapshotter //! feature has to be enabled in the docker daemon use std::fmt::{Display, Formatter}; +use std::sync::Arc; use std::{collections::HashMap, path::Path}; use async_trait::async_trait; -use cli::CommandLine; use crane::CraneCLI; -use krane_bundle::KRANE; use olpc_cjson::CanonicalFormatter; use serde::{Deserialize, Serialize}; use snafu::ResultExt; -mod cli; mod crane; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ImageTool { - image_tool_impl: Box, + image_tool_impl: Arc, } impl ImageTool { - /// Uses the builtin `krane` provided by the `tools/krane` crate. - pub fn from_builtin_krane() -> Self { - let image_tool_impl = Box::new(CraneCLI { - cli: CommandLine { - path: KRANE.path().to_path_buf(), - }, - }); - Self { image_tool_impl } + /// Creates a new `ImageTool` using a statically linked `krane`. + pub fn krane() -> Self { + Self::new(Arc::new(CraneCLI)) } - pub fn new(image_tool_impl: Box) -> Self { + pub fn new(image_tool_impl: Arc) -> Self { Self { image_tool_impl } } @@ -157,6 +150,7 @@ pub mod error { use std::path::PathBuf; use snafu::Snafu; + use tokio::task::JoinError; #[derive(Snafu, Debug)] #[snafu(visibility(pub(super)))] @@ -167,21 +161,21 @@ pub mod error { #[snafu(display("Failed to read archive: {source}"))] ArchiveRead { source: std::io::Error }, - #[snafu(display("Failed to execute image tool, {message}: {source}"))] - CommandFailed { - message: String, - source: std::io::Error, - }, - #[snafu(display("Failed to deserialize image config: {source}"))] ConfigDeserialize { source: serde_json::Error }, #[snafu(display("Failed to create temporary directory for crane push: {source}"))] CraneTemp { source: std::io::Error }, + #[snafu(display("Failed to call crane via FFI: {source}"))] + CraneFFI { source: krane_static::KraneError }, + #[snafu(display("Failed to create temporary directory for docker save: {source}"))] DockerTemp { source: std::io::Error }, + #[snafu(display("Failed to join asynchronous task: {source}"))] + Fork { source: JoinError }, + #[snafu(display("invalid architecture '{value}'"))] InvalidArchitecture { value: String }, @@ -191,21 +185,6 @@ pub mod error { #[snafu(display("Failed to canonicalize image manifest: {source}"))] ManifestCanonicalize { source: serde_json::Error }, - #[snafu(display("No digest returned by `docker load`"))] - NoDigest, - - #[snafu(display( - "Unable to find any supported container image tool, please install docker or crane: {}", - source - ))] - NoneFound { source: which::Error }, - - #[snafu(display( - "Unable to find a container image tool by name '{}' in current environment", - name - ))] - NotFound { name: String, source: which::Error }, - #[snafu(display("Failed to run operation with image tool: {message}\n command: {} {}", program.display(), args.join(" ")))] OperationFailed { message: String, @@ -213,9 +192,6 @@ pub mod error { args: Vec, }, - #[snafu(display("Failed to parse kit filename: {}", source))] - Regex { source: regex::Error }, - #[snafu(display("Unsupported container image tool '{}'", name))] Unsupported { name: String }, } diff --git a/tools/pubsys/src/kit/publish_kit/mod.rs b/tools/pubsys/src/kit/publish_kit/mod.rs index 07aff00a..722b4986 100644 --- a/tools/pubsys/src/kit/publish_kit/mod.rs +++ b/tools/pubsys/src/kit/publish_kit/mod.rs @@ -31,7 +31,7 @@ pub(crate) struct PublishKitArgs { } pub(crate) async fn run(args: &Args, publish_kit_args: &PublishKitArgs) -> Result<()> { - let image_tool = ImageTool::from_builtin_krane(); + let image_tool = ImageTool::krane(); // If a lock file exists, use that, otherwise use Infra.toml let infra_config = InfraConfig::from_path_or_lock(&args.infra_config_path, false) diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 1aaf6c05..27d8d7c6 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -21,7 +21,7 @@ env_logger.workspace = true filetime.workspace = true flate2.workspace = true futures.workspace = true -krane-bundle.workspace = true +krane-static.workspace = true log.workspace = true oci-cli-wrapper.workspace = true olpc-cjson.workspace = true diff --git a/twoliter/src/project/lock/mod.rs b/twoliter/src/project/lock/mod.rs index 6d9a07b9..f5a763c9 100644 --- a/twoliter/src/project/lock/mod.rs +++ b/twoliter/src/project/lock/mod.rs @@ -103,10 +103,9 @@ impl LockedSDK { }; debug!(?sdk, "Resolving workspace SDK"); - let image_tool = ImageTool::from_builtin_krane(); ImageResolver::from_image(&sdk)? .skip_metadata_retrieval() // SDKs don't have metadata - .resolve(&image_tool) + .resolve(&ImageTool::krane()) .await .map(|(sdk, _)| Some(Self(sdk))) } @@ -203,7 +202,6 @@ impl Lock { /// Fetches all external kits defined in a Twoliter.lock to the build directory #[instrument(level = "trace", skip_all)] pub(crate) async fn fetch(&self, project: &Project, arch: &str) -> Result<()> { - let image_tool = ImageTool::from_builtin_krane(); let target_dir = project.external_kits_dir(); create_dir_all(&target_dir).await.context(format!( "failed to create external-kits directory at {}", @@ -218,7 +216,7 @@ impl Lock { let image = project.as_project_image(image)?; let resolver = ImageResolver::from_image(&image)?; resolver - .extract(&image_tool, &project.external_kits_dir(), arch) + .extract(&ImageTool::krane(), &project.external_kits_dir(), arch) .await?; } @@ -257,7 +255,6 @@ impl Lock { async fn resolve(project: &Project) -> Result { let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); - let image_tool = ImageTool::from_builtin_krane(); let mut remaining = project.direct_kit_deps()?; let mut sdk_set = HashSet::new(); @@ -292,7 +289,7 @@ impl Lock { image.version().clone(), ); let image_resolver = ImageResolver::from_image(image)?; - let (locked_image, metadata) = image_resolver.resolve(&image_tool).await?; + let (locked_image, metadata) = image_resolver.resolve(&ImageTool::krane()).await?; let metadata = metadata.context(format!( "failed to validate kit image with name {} from vendor {}", locked_image.name, locked_image.vendor @@ -322,7 +319,7 @@ impl Lock { debug!(?sdk, "Resolving workspace SDK"); let (sdk, _metadata) = ImageResolver::from_image(sdk)? .skip_metadata_retrieval() // SDKs don't have metadata - .resolve(&image_tool) + .resolve(&ImageTool::krane()) .await?; Ok(Self { diff --git a/twoliter/src/tools.rs b/twoliter/src/tools.rs index fe5b1c32..09053db8 100644 --- a/twoliter/src/tools.rs +++ b/twoliter/src/tools.rs @@ -2,7 +2,6 @@ use crate::common::fs; use anyhow::{Context, Result}; use filetime::{set_file_handle_times, set_file_mtime, FileTime}; use flate2::read::ZlibDecoder; -use krane_bundle::KRANE; use std::path::Path; use tar::Archive; use tokio::fs::OpenOptions; @@ -50,7 +49,6 @@ pub(crate) async fn install_tools(tools_dir: impl AsRef) -> Result<()> { write_bin("testsys", TESTSYS, &dir, mtime).await?; write_bin("tuftool", TUFTOOL, &dir, mtime).await?; write_bin("unplug", UNPLUG, &dir, mtime).await?; - fs::copy(KRANE.path(), dir.join("krane")).await?; // Apply the mtime to the directory now that the writes are done. set_file_mtime(dir, mtime).context(format!("Unable to set mtime for '{}'", dir.display()))?; From cb4d91b630ca4f781085c21866e057e025f0bb35 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Mon, 23 Dec 2024 18:25:21 +0000 Subject: [PATCH 2/5] twoliter: require Docker 23 Docker versions older than 23 no longer receive community-supported security patches. Additionally, twoliter requires dockerfile syntax 1.4.3. Currently the tool pulls this image from the network at build time; however, Docker 23 bundles this version internally via buildkit. We can further isolate twoliter builds from the network by ensuring we are on Docker 23 or greater. --- Cargo.lock | 38 +++++++++++++- Cargo.toml | 1 + twoliter/Cargo.toml | 5 ++ twoliter/embedded/build.Dockerfile | 6 ++- twoliter/src/docker/commands.rs | 23 +++++++++ twoliter/src/docker/mod.rs | 1 + twoliter/src/main.rs | 3 ++ twoliter/src/preflight.rs | 80 ++++++++++++++++++++++++++++++ 8 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 twoliter/src/preflight.rs diff --git a/Cargo.lock b/Cargo.lock index 2039fada..76529523 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3358,6 +3358,39 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.79", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", + "test-case-core", +] + [[package]] name = "testsys" version = "0.1.0" @@ -3870,6 +3903,7 @@ dependencies = [ "flate2", "futures", "krane-static", + "lazy_static", "log", "oci-cli-wrapper", "olpc-cjson", @@ -3883,6 +3917,7 @@ dependencies = [ "strum", "tar", "tempfile", + "test-case", "testsys", "tokio", "toml", @@ -3890,6 +3925,7 @@ dependencies = [ "tuftool", "unplug", "uuid", + "which", ] [[package]] @@ -4195,7 +4231,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 82a10165..1e63d0ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ tabled = "0.10" tar = "0.4" tempfile = "3" term_size = "0.3" +test-case = "3" tinytemplate = "1" tokio = "1" tokio-stream = "0.1" diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 27d8d7c6..b5d06f33 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -22,6 +22,7 @@ filetime.workspace = true flate2.workspace = true futures.workspace = true krane-static.workspace = true +lazy_static.workspace = true log.workspace = true oci-cli-wrapper.workspace = true olpc-cjson.workspace = true @@ -36,6 +37,7 @@ tokio = { workspace = true, features = ["fs", "macros", "process", "rt-multi-thr toml.workspace = true tracing = { workspace = true, features = ["log"] } uuid = { workspace = true, features = ["v4"] } +which.workspace = true # Binary dependencies. These are binaries that we want to embed in the Twoliter binary buildsys = { workspace = true } @@ -51,6 +53,9 @@ bytes.workspace = true flate2.workspace = true tar.workspace = true +[dev-dependencies] +test-case.workspace = true + [features] default = ["integ-tests"] integ-tests = [] diff --git a/twoliter/embedded/build.Dockerfile b/twoliter/embedded/build.Dockerfile index 0bfad6cd..8f53dd52 100644 --- a/twoliter/embedded/build.Dockerfile +++ b/twoliter/embedded/build.Dockerfile @@ -1,4 +1,8 @@ -# syntax=docker/dockerfile:1.4.3 +# Twoliter requires minimum Docker version 23.0.0, which ships with a bundled syntax image 1.4.3 +# We refrain from an explicit `syntax` directive as this can lead to unwanted network requests at +# build time. +# See https://hub.docker.com/r/docker/dockerfile for more information + # This Dockerfile has three sections which are used to build rpm.spec packages, to create # kits, and to create Bottlerocket images, respectively. They are marked as Sections 1-3. # buildsys uses Section 1 during build-package calls, Section 2 during build-kit calls, diff --git a/twoliter/src/docker/commands.rs b/twoliter/src/docker/commands.rs index 8b137891..866eda1f 100644 --- a/twoliter/src/docker/commands.rs +++ b/twoliter/src/docker/commands.rs @@ -1 +1,24 @@ +use crate::common::exec; +use anyhow::{Context, Result}; +use semver::Version; +use tokio::process::Command; +pub(crate) struct Docker; + +impl Docker { + /// Fetches the version of the docker daemon + pub(crate) async fn server_version() -> Result { + let version_str = exec( + Command::new("docker").args(["version", "--format", "{{.Server.Version}}"]), + true, + ) + .await + // Convert Result> to Option + .ok() + .flatten() + .map(|s| s.trim().to_string()) + .context("Failed to fetch docker version")?; + + Version::parse(&version_str).context("Failed to parse docker version as semver") + } +} diff --git a/twoliter/src/docker/mod.rs b/twoliter/src/docker/mod.rs index 94e6f775..1c2c1623 100644 --- a/twoliter/src/docker/mod.rs +++ b/twoliter/src/docker/mod.rs @@ -2,3 +2,4 @@ mod commands; mod image; pub(crate) use self::image::ImageUri; +pub(crate) use commands::Docker; diff --git a/twoliter/src/main.rs b/twoliter/src/main.rs index ed55a2d2..66a06088 100644 --- a/twoliter/src/main.rs +++ b/twoliter/src/main.rs @@ -3,10 +3,12 @@ use anyhow::Result; use clap::Parser; mod cargo_make; +pub(crate) mod cleanup; mod cmd; mod common; mod compatibility; mod docker; +mod preflight; mod project; mod schema_version; /// Test code that should only be compiled when running tests. @@ -20,5 +22,6 @@ mod tools; async fn main() -> Result<()> { let args = Args::parse(); init_logger(args.log_level); + preflight::preflight().await?; cmd::run(args).await } diff --git a/twoliter/src/preflight.rs b/twoliter/src/preflight.rs new file mode 100644 index 00000000..576a9a3e --- /dev/null +++ b/twoliter/src/preflight.rs @@ -0,0 +1,80 @@ +//! This module performs checks that the current environment is compatible with twoliter, as well +//! as any other "global" setup that must occur before the build process begins. +use anyhow::{ensure, Result}; +use lazy_static::lazy_static; +use semver::{Comparator, Op, Prerelease, VersionReq}; +use which::which_global; + +use crate::docker::Docker; + +const REQUIRED_TOOLS: &[&str] = &["docker", "gzip", "lz4"]; + +lazy_static! { + // Twoliter relies on minimum Dockerfile syntax 1.4.3, which is shipped in Docker 23.0.0 by default + // We do not use explicit `syntax=` directives to avoid network connections during the build. + static ref MINIMUM_DOCKER_VERSION: VersionReq = VersionReq { + comparators: [ + Comparator { + op: Op::GreaterEq, + major: 23, + minor: None, + patch: None, + pre: Prerelease::default(), + } + ].into() + }; +} + +/// Runs all common setup required for twoliter. +/// +/// * Ensures that any required system tools are installed an accessible. +/// * Sets up interrupt handler to cleanup on SIGINT +pub(crate) async fn preflight() -> Result<()> { + check_environment().await?; + + Ok(()) +} + +pub(crate) async fn check_environment() -> Result<()> { + check_for_required_tools()?; + check_docker_version().await?; + + Ok(()) +} + +fn check_for_required_tools() -> Result<()> { + for tool in REQUIRED_TOOLS { + ensure!( + which_global(tool).is_ok(), + "Failed to find required tool `{tool}` in PATH" + ); + } + Ok(()) +} + +async fn check_docker_version() -> Result<()> { + let docker_version = Docker::server_version().await?; + + ensure!( + MINIMUM_DOCKER_VERSION.matches(&docker_version), + "docker found in PATH does not meet the minimum version requirements for twoliter: {}", + MINIMUM_DOCKER_VERSION.to_string(), + ); + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + use semver::Version; + use test_case::test_case; + + #[test_case(Version::parse("25.0.5").unwrap(), true; "25.0.5 passes")] + #[test_case(Version::parse("27.1.4").unwrap(), true; "27.1.4 passes")] + #[test_case(Version::parse("18.0.9").unwrap(), false; "18.0.9 fails")] + #[test_case(Version::parse("20.10.27").unwrap(), false)] + fn test_docker_version_req(version: Version, is_ok: bool) { + assert_eq!(MINIMUM_DOCKER_VERSION.matches(&version), is_ok) + } +} From ecdaa4361411e30c23bfa9b82539f16cd594e198 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Mon, 23 Dec 2024 19:04:46 +0000 Subject: [PATCH 3/5] twoliter: refactor project to reduce module length The project module is large, making it difficult to organize and read. This moves much of the vended image/artifact machinery into a new module. We also add a new trait to help us write functionality over both Projects that are fully locked and Projects with a locked SDK. --- twoliter/src/project/image.rs | 195 +++++++++++++++++++++++++++ twoliter/src/project/lock/mod.rs | 3 +- twoliter/src/project/mod.rs | 223 ++++--------------------------- 3 files changed, 222 insertions(+), 199 deletions(-) create mode 100644 twoliter/src/project/image.rs diff --git a/twoliter/src/project/image.rs b/twoliter/src/project/image.rs new file mode 100644 index 00000000..4933a396 --- /dev/null +++ b/twoliter/src/project/image.rs @@ -0,0 +1,195 @@ +//! Contains abstractions representing image artifacts referred by a Project. +use super::ArtifactVendor; +use crate::docker::ImageUri; +use anyhow::{ensure, Result}; +use semver::Version; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Debug, Display, Formatter}; +use std::hash::Hash; +use std::str::FromStr; + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub(crate) struct ProjectImage { + pub(super) image: Image, + pub(super) vendor: ArtifactVendor, +} + +impl Display for ProjectImage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.vendor { + ArtifactVendor::Overridden(_) => write!( + f, + "{}-{}@{} (overridden-to: {})", + self.name(), + self.version(), + self.original_source_uri(), + self.project_image_uri(), + ), + ArtifactVendor::Verbatim(_) => write!( + f, + "{}-{}@{}", + self.name(), + self.version(), + self.original_source_uri() + ), + } + } +} + +impl ProjectImage { + pub(crate) fn name(&self) -> &ValidIdentifier { + &self.image.name + } + + pub(crate) fn version(&self) -> &Version { + self.image.version() + } + + pub(crate) fn vendor_name(&self) -> &ValidIdentifier { + self.vendor.vendor_name() + } + + /// Returns the URI for the original vendor. + pub(crate) fn original_source_uri(&self) -> ImageUri { + match &self.vendor { + ArtifactVendor::Overridden(overridden) => { + let original = ArtifactVendor::Verbatim(overridden.original_vendor()); + original.image_uri_for(&self.image) + } + ArtifactVendor::Verbatim(_) => self.vendor.image_uri_for(&self.image), + } + } + + /// Returns the image URI that the project will use for this image + /// + /// This could be different than the source_uri if overridden. + pub(crate) fn project_image_uri(&self) -> ImageUri { + ImageUri { + registry: Some(self.vendor.registry().to_string()), + repo: self.vendor.repo_for(&self.image).to_string(), + tag: format!("v{}", self.image.version()), + } + } +} + +/// An artifact/vendor name combination used to identify an artifact resolved by Twoliter. +/// +/// This is intended for use in [`Project::vendor_for`] lookups. +pub(crate) trait VendedArtifact: std::fmt::Debug { + fn artifact_name(&self) -> &ValidIdentifier; + fn vendor_name(&self) -> &ValidIdentifier; + fn version(&self) -> &Version; +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct ValidIdentifier(pub(crate) String); + +impl Serialize for ValidIdentifier { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + serializer.serialize_str(self.0.as_str()) + } +} + +impl FromStr for ValidIdentifier { + type Err = anyhow::Error; + + fn from_str(input: &str) -> Result { + ensure!( + !input.is_empty(), + "cannot define an identifier as an empty string", + ); + + // Check if the input contains any invalid characters + for c in input.chars() { + ensure!( + is_valid_id_char(c), + "invalid character '{}' found in identifier name", + c + ); + } + + Ok(Self(input.to_string())) + } +} + +impl<'de> Deserialize<'de> for ValidIdentifier { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let input = String::deserialize(deserializer)?; + input.parse().map_err(D::Error::custom) + } +} + +impl AsRef for ValidIdentifier { + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + +impl Display for ValidIdentifier { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0.as_str()) + } +} + +fn is_valid_id_char(c: char) -> bool { + match c { + // Allow alphanumeric characters, underscores, and hyphens + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => true, + // Disallow other characters + _ => false, + } +} + +/// This represents a container registry vendor that is used in resolving the kits and also +/// now the bottlerocket sdk +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Vendor { + pub registry: String, +} + +/// This represents a dependency on a container, primarily used for kits +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Image { + pub name: ValidIdentifier, + pub version: Version, + pub vendor: ValidIdentifier, +} + +impl Image { + pub(super) fn from_vended_artifact(artifact: &impl VendedArtifact) -> Self { + Self { + name: artifact.artifact_name().clone(), + vendor: artifact.vendor_name().clone(), + version: artifact.version().clone(), + } + } +} + +impl Display for Image { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}-{}@{}", self.name, self.version, self.vendor) + } +} + +impl VendedArtifact for Image { + fn artifact_name(&self) -> &ValidIdentifier { + &self.name + } + + fn vendor_name(&self) -> &ValidIdentifier { + &self.vendor + } + + fn version(&self) -> &Version { + &self.version + } +} diff --git a/twoliter/src/project/lock/mod.rs b/twoliter/src/project/lock/mod.rs index f5a763c9..5cdd62d5 100644 --- a/twoliter/src/project/lock/mod.rs +++ b/twoliter/src/project/lock/mod.rs @@ -13,12 +13,13 @@ mod verification; mod views; pub(crate) use self::verification::VerificationTagger; +pub(crate) use image::LockedImage; use crate::common::fs::{create_dir_all, read, write}; use crate::project::{Project, ValidIdentifier}; use crate::schema_version::SchemaVersion; use anyhow::{bail, ensure, Context, Result}; -use image::{ImageResolver, LockedImage}; +use image::ImageResolver; use oci_cli_wrapper::ImageTool; use olpc_cjson::CanonicalFormatter as CanonicalJsonFormatter; use semver::Version; diff --git a/twoliter/src/project/mod.rs b/twoliter/src/project/mod.rs index 8fe87f5f..a33772f2 100644 --- a/twoliter/src/project/mod.rs +++ b/twoliter/src/project/mod.rs @@ -1,13 +1,14 @@ +mod image; mod lock; pub(crate) mod vendor; +pub(crate) use self::image::{Image, ProjectImage, ValidIdentifier, VendedArtifact, Vendor}; pub(crate) use self::vendor::ArtifactVendor; pub(crate) use lock::VerificationTagger; use self::lock::{Lock, LockedSDK, Override}; use crate::common::fs::{self, read_to_string}; use crate::compatibility::SUPPORTED_TWOLITER_PROJECT_SCHEMA_VERSION; -use crate::docker::ImageUri; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; @@ -15,15 +16,11 @@ use async_trait::async_trait; use async_walkdir::WalkDir; use buildsys_config::{EXTERNAL_KIT_DIRECTORY, EXTERNAL_KIT_METADATA}; use futures::stream::StreamExt; -use semver::Version; -use serde::de::Error; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::Deserialize; use std::collections::BTreeMap; use std::ffi::OsStr; -use std::fmt::{Debug, Display, Formatter}; -use std::hash::Hash; +use std::fmt::Debug; use std::path::{Path, PathBuf}; -use std::str::FromStr; use toml::Table; use tracing::{debug, info, instrument, trace, warn}; @@ -298,197 +295,6 @@ impl Project { .collect::>() .expect("Could not find kit vendor despite lock resolution succeeding?") } - - pub(crate) fn sdk_image(&self) -> ProjectImage { - let Locked(lock) = &self.lock; - self.as_project_image(&lock.sdk) - .expect("Could not find SDK vendor despite lock resolution succeeding?") - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub(crate) struct ProjectImage { - image: Image, - vendor: ArtifactVendor, -} - -impl Display for ProjectImage { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self.vendor { - ArtifactVendor::Overridden(_) => write!( - f, - "{}-{}@{} (overridden-to: {})", - self.name(), - self.version(), - self.original_source_uri(), - self.project_image_uri(), - ), - ArtifactVendor::Verbatim(_) => write!( - f, - "{}-{}@{}", - self.name(), - self.version(), - self.original_source_uri() - ), - } - } -} - -impl ProjectImage { - pub(crate) fn name(&self) -> &ValidIdentifier { - &self.image.name - } - - pub(crate) fn version(&self) -> &Version { - self.image.version() - } - - pub(crate) fn vendor_name(&self) -> &ValidIdentifier { - self.vendor.vendor_name() - } - - /// Returns the URI for the original vendor. - pub(crate) fn original_source_uri(&self) -> ImageUri { - match &self.vendor { - ArtifactVendor::Overridden(overridden) => { - let original = ArtifactVendor::Verbatim(overridden.original_vendor()); - original.image_uri_for(&self.image) - } - ArtifactVendor::Verbatim(_) => self.vendor.image_uri_for(&self.image), - } - } - - /// Returns the image URI that the project will use for this image - /// - /// This could be different than the source_uri if overridden. - pub(crate) fn project_image_uri(&self) -> ImageUri { - ImageUri { - registry: Some(self.vendor.registry().to_string()), - repo: self.vendor.repo_for(&self.image).to_string(), - tag: format!("v{}", self.image.version()), - } - } -} - -/// An artifact/vendor name combination used to identify an artifact resolved by Twoliter. -/// -/// This is intended for use in [`Project::vendor_for`] lookups. -pub(crate) trait VendedArtifact: std::fmt::Debug { - fn artifact_name(&self) -> &ValidIdentifier; - fn vendor_name(&self) -> &ValidIdentifier; - fn version(&self) -> &Version; -} - -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub(crate) struct ValidIdentifier(pub(crate) String); - -impl Serialize for ValidIdentifier { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - serializer.serialize_str(self.0.as_str()) - } -} - -impl FromStr for ValidIdentifier { - type Err = anyhow::Error; - - fn from_str(input: &str) -> Result { - ensure!( - !input.is_empty(), - "cannot define an identifier as an empty string", - ); - - // Check if the input contains any invalid characters - for c in input.chars() { - ensure!( - is_valid_id_char(c), - "invalid character '{}' found in identifier name", - c - ); - } - - Ok(Self(input.to_string())) - } -} - -impl<'de> Deserialize<'de> for ValidIdentifier { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let input = String::deserialize(deserializer)?; - input.parse().map_err(D::Error::custom) - } -} - -impl AsRef for ValidIdentifier { - fn as_ref(&self) -> &str { - self.0.as_str() - } -} - -impl Display for ValidIdentifier { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(self.0.as_str()) - } -} - -fn is_valid_id_char(c: char) -> bool { - match c { - // Allow alphanumeric characters, underscores, and hyphens - 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => true, - // Disallow other characters - _ => false, - } -} - -/// This represents a container registry vendor that is used in resolving the kits and also -/// now the bottlerocket sdk -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] -#[serde(rename_all = "kebab-case")] -pub(crate) struct Vendor { - pub registry: String, -} - -/// This represents a dependency on a container, primarily used for kits -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] -#[serde(rename_all = "kebab-case")] -pub(crate) struct Image { - pub name: ValidIdentifier, - pub version: Version, - pub vendor: ValidIdentifier, -} - -impl Image { - fn from_vended_artifact(artifact: &impl VendedArtifact) -> Self { - Self { - name: artifact.artifact_name().clone(), - vendor: artifact.vendor_name().clone(), - version: artifact.version().clone(), - } - } -} - -impl Display for Image { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}-{}@{}", self.name, self.version, self.vendor) - } -} - -impl VendedArtifact for Image { - fn artifact_name(&self) -> &ValidIdentifier { - &self.name - } - - fn vendor_name(&self) -> &ValidIdentifier { - &self.vendor - } - - fn version(&self) -> &Version { - &self.version - } } /// This is used to `Deserialize` a project, then run validation code before returning a valid @@ -687,6 +493,25 @@ impl From for Locked { } } +/// A trait representing Projects which have verified their locked SDK. +pub(crate) trait LockedSDKProvider: ProjectLock { + fn locked_sdk_image(&self) -> &LockedImage; +} + +impl LockedSDKProvider for SDKLocked { + fn locked_sdk_image(&self) -> &LockedImage { + let SDKLocked(lock) = self; + &lock.0 + } +} + +impl LockedSDKProvider for Locked { + fn locked_sdk_image(&self) -> &LockedImage { + let Locked(lock) = self; + &lock.sdk + } +} + /// Seal the `ProjectLock` trait -- only this module is allowed to define new lock types. mod private { /// A marker type that, when used in a method signature, makes it impossible for other modules @@ -698,7 +523,9 @@ mod private { mod test { use super::*; use crate::common::fs; + use crate::docker::ImageUri; use crate::test::{data_dir, projects_dir}; + use semver::Version; use tempfile::TempDir; /// Ensure that `Twoliter.toml` can be deserialized. From faac675f1c594e7c1f037561e032b906ae27534d Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Mon, 23 Dec 2024 19:26:28 +0000 Subject: [PATCH 4/5] twoliter: fetch build SDK prior to cargo-make Now that `krane` is run as a static library, we cannot execute it within Makefile.toml. This change moves the SDK fetch to occur prior to executing build make targets. --- Cargo.lock | 11 +++++ Cargo.toml | 1 + twoliter/Cargo.toml | 1 + twoliter/embedded/Makefile.toml | 69 +++----------------------- twoliter/src/cleanup.rs | 86 +++++++++++++++++++++++++++++++++ twoliter/src/cmd/build.rs | 2 + twoliter/src/cmd/fetch.rs | 3 +- twoliter/src/cmd/make.rs | 44 ++++++----------- twoliter/src/cmd/publish_kit.rs | 2 + twoliter/src/docker/commands.rs | 52 +++++++++++++++++++- twoliter/src/preflight.rs | 7 ++- twoliter/src/project/mod.rs | 13 +++-- twoliter/src/project/tasks.rs | 56 +++++++++++++++++++++ 13 files changed, 249 insertions(+), 98 deletions(-) create mode 100644 twoliter/src/cleanup.rs create mode 100644 twoliter/src/project/tasks.rs diff --git a/Cargo.lock b/Cargo.lock index 76529523..2c0774f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,6 +1067,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys 0.52.0", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -3898,6 +3908,7 @@ dependencies = [ "buildsys-config", "bytes", "clap", + "ctrlc", "env_logger", "filetime", "flate2", diff --git a/Cargo.toml b/Cargo.toml index 1e63d0ad..1f96bfb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ bytes = "1" chrono = { version = "0.4", default-features = false } clap = "4" coldsnap = { version = "0.6", default-features = false } +ctrlc = "3" daemonize = "0.5" duct = "0.13" env_logger = "0.11" diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index b5d06f33..55bbd3eb 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -17,6 +17,7 @@ async-trait.workspace = true base64.workspace = true buildsys-config.workspace = true clap = { workspace = true, features = ["derive", "env", "std"] } +ctrlc = { workspace = true, features = ["termination"] } env_logger.workspace = true filetime.workspace = true flate2.workspace = true diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index e7b50202..168a0836 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -13,7 +13,6 @@ BUILDSYS_BUILD_DIR = "${BUILDSYS_ROOT_DIR}/build" BUILDSYS_PACKAGES_DIR = "${BUILDSYS_BUILD_DIR}/rpms" BUILDSYS_KITS_DIR = "${BUILDSYS_BUILD_DIR}/kits" BUILDSYS_EXTERNAL_KITS_DIR = "${BUILDSYS_BUILD_DIR}/external-kits" -BUILDSYS_EXTERNAL_SDKS_DIR = "${BUILDSYS_BUILD_DIR}/external-sdk-archives" BUILDSYS_STATE_DIR = "${BUILDSYS_BUILD_DIR}/state" BUILDSYS_IMAGES_DIR = "${BUILDSYS_BUILD_DIR}/images" BUILDSYS_LOGS_DIR = "${BUILDSYS_BUILD_DIR}/logs" @@ -291,64 +290,12 @@ mkdir -p ${GO_MOD_CACHE} ''' ] -[tasks.setup-build] -dependencies = ["setup"] -script = [ -''' -for cmd in docker gzip lz4; do - if ! command -v ${cmd} >/dev/null 2>&1 ; then - echo "required program '${cmd}' not found" >&2 - exit 1 - fi -done -''' -] - [tasks.fetch] dependencies = [ - "fetch-sdk", "fetch-sources", "fetch-vendored", ] -[tasks.fetch-sdk] -dependencies = ["setup-build"] -script_runner = "bash" -script = [ -''' - -cleanup() { - [ -n "${SDK_ARCHIVE_PATH}" ] && rm -rf "${SDK_ARCHIVE_PATH}" -} - -trap 'cleanup' EXIT - -SDK_PLATFORM="$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}')" -KRANE="${TWOLITER_TOOLS_DIR}/krane" - -mkdir -p "${BUILDSYS_EXTERNAL_SDKS_DIR}" -SDK_ARCHIVE_PATH="$(mktemp -p ${BUILDSYS_EXTERNAL_SDKS_DIR} bottlerocket-sdk-tmp-archive-XXXXXXXX.tar)" - -if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.sdk-verified" ]; then - echo "Twoliter could not validate '${TLPRIVATE_SDK_IMAGE}', refusing to continue" >&2 - exit 1 -fi - -if ! docker image inspect "${TLPRIVATE_SDK_IMAGE}" >/dev/null 2>&1 ; then - echo "Pulling SDK '${TLPRIVATE_SDK_IMAGE}'" - if ! ${KRANE} pull "${TLPRIVATE_SDK_IMAGE}" "${SDK_ARCHIVE_PATH}" --platform "${SDK_PLATFORM}" ; then - echo "failed to pull '${TLPRIVATE_SDK_IMAGE}'" >&2 - exit 1 - fi - - if ! docker load --input "${SDK_ARCHIVE_PATH}" ; then - echo "failed to load '${TLPRIVATE_SDK_IMAGE}' into docker daemon" >&2 - exit 1 - fi -fi -''' -] - [tasks.fetch-sources] dependencies = ["setup"] script_runner = "bash" @@ -364,7 +311,7 @@ chmod -R o+r ${CARGO_HOME} ] [tasks.fetch-vendored] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script = [ ''' go_fetch() { @@ -384,7 +331,7 @@ done ] [tasks.unit-tests] -dependencies = ["fetch-sdk", "fetch-sources", "fetch-vendored"] +dependencies = ["setup", "fetch-sources", "fetch-vendored"] script = [ ''' export VARIANT="${BUILDSYS_VARIANT}" @@ -639,7 +586,7 @@ fi ] [tasks.build-sbkeys] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -707,7 +654,7 @@ fi ] [tasks.boot-config] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -747,7 +694,7 @@ echo "Boot configuration initrd may be found at ${BOOT_CONFIG}" ] [tasks.validate-boot-config] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -908,7 +855,7 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${BUILDSYS_OUTPUT_DIR}/latest" ] [tasks.repack-variant] -dependencies = ["fetch-sdk", "build-sbkeys", "publish-setup", "cargo-metadata"] +dependencies = ["setup", "build-sbkeys", "publish-setup", "cargo-metadata"] script = [ ''' export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" @@ -1223,7 +1170,7 @@ pubsys \ ] [tasks.ami] -dependencies = ["setup-build"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -1569,7 +1516,7 @@ pubsys \ ] [tasks._upload-ova-base] -dependencies = ["setup-build"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' diff --git a/twoliter/src/cleanup.rs b/twoliter/src/cleanup.rs new file mode 100644 index 00000000..046fe1c1 --- /dev/null +++ b/twoliter/src/cleanup.rs @@ -0,0 +1,86 @@ +//! Provides a mechanism for cleaning up resources when twoliter is interrupted. + +use anyhow::{Context, Result}; +use std::collections::BTreeMap; +use std::future::Future; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use tempfile::TempPath; +use uuid::Uuid; + +use self::sealed::*; + +lazy_static::lazy_static! { + pub(crate) static ref JANITOR: TempfileJanitor = TempfileJanitor::default(); +} + +impl TempfileJanitor { + /// Run a given async closure using a [`tempfile::TempPath`]. + /// + /// The closure has access to the path where the (closed) tempfile is stored. + /// [`TempfileJanitor`] will ensure that the temporary file is deleted in the case that the + /// current process receives SIGINT/SIGTERM/SIGHUP. + pub(crate) async fn with_tempfile( + &self, + tmpfile: TempPath, + do_: impl FnOnce(PathBuf) -> Fut, + ) -> Result + where + Fut: Future, + { + let path = tmpfile.to_path_buf(); + let path_id = Uuid::new_v4(); + + self.paths.lock().unwrap().insert(path_id, tmpfile); + + let result = do_(path).await; + + self.paths.lock().unwrap().remove(&path_id); + + Ok(result) + } + + pub(crate) fn try_cleanup(&mut self) { + tracing::info!("Cleaning up temporary resources..."); + if let Ok(mut paths) = self.paths.lock() { + while let Some((_, path)) = paths.pop_first() { + tracing::debug!("Deleting tempfile at '{}'", path.display()); + if let Err(e) = std::fs::remove_file(&path) { + tracing::error!("Failed to clean tempfile '{}': {}", path.display(), e); + } + } + } + tracing::info!("Done cleaning up."); + } + + /// Attempts to install the cleanup process as a SIGINT/SIGTERM/SIGHUP signal handler + pub(crate) fn setup_signal_handler(&self) -> Result<()> { + let mut handler_ref = Self { + paths: Arc::clone(&self.paths), + }; + + let already_handling = Arc::new(AtomicBool::new(false)); + ctrlc::try_set_handler(move || { + if already_handling + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + handler_ref.try_cleanup(); + } + // SIGINT is 130 + std::process::exit(130); + }) + .context("Failed to create cleanup signal handler") + } +} + +/// Signal handlers are global -- hide `TempfileJanitor` to encourage use of the static reference. +mod sealed { + use super::*; + + #[derive(Default, Debug)] + pub(crate) struct TempfileJanitor { + pub(super) paths: Arc>>, + } +} diff --git a/twoliter/src/cmd/build.rs b/twoliter/src/cmd/build.rs index b75e836d..4666fcdf 100644 --- a/twoliter/src/cmd/build.rs +++ b/twoliter/src/cmd/build.rs @@ -63,6 +63,7 @@ impl BuildKit { optional_envs.push(("BUILDSYS_LOOKASIDE_CACHE", lookaside_cache)) } + project.fetch_sdk().await?; CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_ARCH", &self.arch) @@ -135,6 +136,7 @@ impl BuildVariant { )) } + project.fetch_sdk().await?; CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_ARCH", &self.arch) diff --git a/twoliter/src/cmd/fetch.rs b/twoliter/src/cmd/fetch.rs index 1e3c9918..80a656f1 100644 --- a/twoliter/src/cmd/fetch.rs +++ b/twoliter/src/cmd/fetch.rs @@ -18,7 +18,8 @@ impl Fetch { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; let project = project.load_lock::().await?; - project.fetch(self.arch.as_str()).await?; + project.fetch_kits(self.arch.as_str()).await?; + project.fetch_sdk().await?; Ok(()) } } diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index 23b80ccf..75f11c7d 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -54,7 +54,19 @@ pub(crate) struct Make { impl Make { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let sdk_source = self.locked_sdk(&project).await?; + + let sdk_source = if self.can_skip_kit_verification(&project) { + let project = project.load_lock::().await?; + project.fetch_sdk().await?; + project.sdk_image() + } else { + let project = project.load_lock::().await?; + project.fetch_sdk().await?; + project.sdk_image() + } + .project_image_uri() + .to_string(); + let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); @@ -75,17 +87,6 @@ impl Make { target_allows_kit_verification_skip && project_has_explicit_sdk_dep } - - /// Returns the locked SDK image for the project. - async fn locked_sdk(&self, project: &project::Project) -> Result { - Ok(if self.can_skip_kit_verification(project) { - project.load_lock::().await?.sdk_image() - } else { - project.load_lock::().await?.sdk_image() - } - .project_image_uri() - .to_string()) - } } #[cfg(test)] @@ -227,6 +228,7 @@ mod test { install_tools(&toolsdir).await.unwrap(); let makefile_path = toolsdir.join("Makefile.toml"); + project.fetch_sdk().await?; CargoMake::new(&sdk_source) .unwrap() .env("CARGO_HOME", project_dir.display().to_string()) @@ -266,24 +268,6 @@ mod test { assert!(!target_can_skip_kit_verification("build-variant").await); } - #[tokio::test] - #[ignore] // integration test - async fn test_fetch_sdk_succeeds_when_only_sdk_verified() { - let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); - assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), false) - .await - .is_ok()); - } - - #[tokio::test] - #[ignore] // integration test - async fn test_fetch_sdk_fails_when_nothing_verified() { - let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); - assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), true) - .await - .is_err()); - } - #[tokio::test] #[ignore] // integration test async fn test_validate_kits_fails_when_only_sdk_verified() { diff --git a/twoliter/src/cmd/publish_kit.rs b/twoliter/src/cmd/publish_kit.rs index a009f555..75bdf7e8 100644 --- a/twoliter/src/cmd/publish_kit.rs +++ b/twoliter/src/cmd/publish_kit.rs @@ -40,6 +40,7 @@ impl PublishKit { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; let project = project.load_lock::().await?; + let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); @@ -48,6 +49,7 @@ impl PublishKit { Some(kit_repo) => kit_repo, None => &self.kit_name, }; + project.fetch_sdk().await?; CargoMake::new(project.sdk_image().project_image_uri().to_string().as_str())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_KIT", &self.kit_name) diff --git a/twoliter/src/docker/commands.rs b/twoliter/src/docker/commands.rs index 866eda1f..a08d393e 100644 --- a/twoliter/src/docker/commands.rs +++ b/twoliter/src/docker/commands.rs @@ -1,11 +1,61 @@ -use crate::common::exec; +use crate::common::{exec, exec_log}; use anyhow::{Context, Result}; use semver::Version; +use std::path::Path; use tokio::process::Command; +use super::ImageUri; + pub(crate) struct Docker; impl Docker { + /// Loads an image tarball into the docker daemon from the given path + pub(crate) async fn load(path: impl AsRef) -> Result<()> { + exec_log( + Command::new("docker") + .args(["load", "-i"]) + .arg(path.as_ref()), + ) + .await + } + + /// Returns whether or not the docker daemon has cached an image with the given URI locally + pub(crate) async fn image_is_cached(image_uri: &ImageUri) -> Result { + let image_hash = exec( + Command::new("docker") + .args(["images", "-q"]) + .arg(image_uri.uri()), + true, + ) + .await + // Convert Result> to Option + .ok() + .flatten() + .map(|s| s.trim().to_string()) + .with_context(|| { + format!( + "Failed to search docker daemon for image '{}'", + image_uri.uri() + ) + })?; + + Ok(!image_hash.is_empty()) + } + + /// Fetches the host platform in the form $OS/$GOARCH, e.g. linux/arm64 + pub(crate) async fn host_platform() -> Result { + exec( + Command::new("docker").args(["version", "--format", "{{.Server.Os}}/{{.Server.Arch}}"]), + true, + ) + .await + // Convert Result> to Option + .ok() + .flatten() + .map(|s| s.trim().to_string()) + .context("Failed to fetch host platform from docker") + } + /// Fetches the version of the docker daemon pub(crate) async fn server_version() -> Result { let version_str = exec( diff --git a/twoliter/src/preflight.rs b/twoliter/src/preflight.rs index 576a9a3e..48dd511e 100644 --- a/twoliter/src/preflight.rs +++ b/twoliter/src/preflight.rs @@ -3,6 +3,7 @@ use anyhow::{ensure, Result}; use lazy_static::lazy_static; use semver::{Comparator, Op, Prerelease, VersionReq}; +use tracing::warn; use which::which_global; use crate::docker::Docker; @@ -28,9 +29,13 @@ lazy_static! { /// Runs all common setup required for twoliter. /// /// * Ensures that any required system tools are installed an accessible. -/// * Sets up interrupt handler to cleanup on SIGINT +/// * Sets up signal handler to cleanup on SIGINT pub(crate) async fn preflight() -> Result<()> { check_environment().await?; + if let Err(e) = crate::cleanup::JANITOR.setup_signal_handler() { + warn!("Failed to register cleanup signal handler: {:?}", e); + warn!("Twoliter may leak resources if interrupted abruptly."); + } Ok(()) } diff --git a/twoliter/src/project/mod.rs b/twoliter/src/project/mod.rs index a33772f2..8863fe82 100644 --- a/twoliter/src/project/mod.rs +++ b/twoliter/src/project/mod.rs @@ -1,9 +1,11 @@ mod image; mod lock; +pub(crate) mod tasks; pub(crate) mod vendor; pub(crate) use self::image::{Image, ProjectImage, ValidIdentifier, VendedArtifact, Vendor}; pub(crate) use self::vendor::ArtifactVendor; +use lock::LockedImage; pub(crate) use lock::VerificationTagger; use self::lock::{Lock, LockedSDK, Override}; @@ -171,6 +173,10 @@ impl Project { self.project_dir.join(EXTERNAL_KIT_METADATA) } + pub(crate) fn external_sdk_archive_dir(&self) -> PathBuf { + self.project_dir.join("build/external-sdk-archives") + } + pub(crate) fn schema_version(&self) -> SchemaVersion<1> { self.schema_version } @@ -271,17 +277,16 @@ impl Project { } } -impl Project { +impl Project { pub(crate) fn sdk_image(&self) -> ProjectImage { - let SDKLocked(lock) = &self.lock; - self.as_project_image(&lock.0) + self.as_project_image(self.lock.locked_sdk_image()) .expect("Could not find SDK vendor despite lock resolution succeeding?") } } impl Project { /// Fetches all external kits defined in a Twoliter.lock to the build directory - pub(crate) async fn fetch(&self, arch: &str) -> Result<()> { + pub(crate) async fn fetch_kits(&self, arch: &str) -> Result<()> { let Locked(lock) = &self.lock; lock.fetch(self, arch).await } diff --git a/twoliter/src/project/tasks.rs b/twoliter/src/project/tasks.rs new file mode 100644 index 00000000..e39644be --- /dev/null +++ b/twoliter/src/project/tasks.rs @@ -0,0 +1,56 @@ +//! This module defines common atomic build tasks that can be performed with a fully loaded project. +use super::{LockedSDKProvider, Project}; +use crate::cleanup::JANITOR; +use crate::docker::Docker; +use anyhow::{Context, Result}; +use krane_static::call_krane_inherited_io; +use tracing::instrument; + +impl Project { + /// Caches the project's SDK into the docker daemon if an image with the same name/tag is not + /// already cached. + #[instrument(level = "trace")] + pub(crate) async fn fetch_sdk(&self) -> Result<()> { + let sdk_uri = self.sdk_image().project_image_uri(); + tracing::info!("Ensuring project SDK '{sdk_uri}' is cached locally."); + + if Docker::image_is_cached(&sdk_uri).await? { + tracing::debug!("SDK '{sdk_uri}' is cached."); + return Ok(()); + } + + let sdk_archive_dir = self.external_sdk_archive_dir(); + tokio::fs::create_dir_all(&sdk_archive_dir).await?; + + let temp_path = tempfile::Builder::new() + .prefix("bottlerocket-sdk-tmp-ardchive-") + .suffix(".tar") + .tempfile_in(&sdk_archive_dir)? + .into_temp_path(); + + let host_platform = Docker::host_platform().await?; + + JANITOR + .with_tempfile(temp_path, |temp_path| async move { + let path_str = temp_path.to_string_lossy().to_string(); + + tracing::info!("Pulling '{sdk_uri}' for platform '{host_platform}'"); + call_krane_inherited_io(&[ + "pull", + &sdk_uri.uri(), + &path_str, + "--platform", + &host_platform, + ]) + .context("Failed to pull SDK image")?; + + tracing::info!("Loading SDK image '{sdk_uri}' into docker daemon"); + Docker::load(&path_str).await?; + + Ok::<_, anyhow::Error>(()) + }) + .await??; + + Ok(()) + } +} From 5ba62f9096227893d093960740cde2bb3dd5f8c1 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Mon, 23 Dec 2024 19:29:49 +0000 Subject: [PATCH 5/5] twoliter: conditionally compile pubsys While we do not publish twoliter without pubsys support built in, it can speed up development compile times to avoid needing to compile pubsys (which subsequently compiles the AWS SDK for Rust.) --- twoliter/Cargo.toml | 5 +++-- twoliter/src/tools.rs | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 55bbd3eb..abb1bbb2 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -43,7 +43,7 @@ which.workspace = true # Binary dependencies. These are binaries that we want to embed in the Twoliter binary buildsys = { workspace = true } pipesys = { workspace = true } -pubsys = { workspace = true } +pubsys = { workspace = true, optional = true } pubsys-setup = { workspace = true } testsys = { workspace = true } tuftool = { workspace = true } @@ -58,5 +58,6 @@ tar.workspace = true test-case.workspace = true [features] -default = ["integ-tests"] +default = ["integ-tests", "pubsys"] integ-tests = [] +pubsys = ["dep:pubsys"] diff --git a/twoliter/src/tools.rs b/twoliter/src/tools.rs index 09053db8..b22c8de0 100644 --- a/twoliter/src/tools.rs +++ b/twoliter/src/tools.rs @@ -12,6 +12,7 @@ use tracing::debug; const TAR_GZ_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/tools.tar.gz")); const BUILDSYS: &[u8] = include_bytes!(env!("CARGO_BIN_FILE_BUILDSYS")); const PIPESYS: &[u8] = include_bytes!(env!("CARGO_BIN_FILE_PIPESYS")); +#[cfg(feature = "pubsys")] const PUBSYS: &[u8] = include_bytes!(env!("CARGO_BIN_FILE_PUBSYS")); const PUBSYS_SETUP: &[u8] = include_bytes!(env!("CARGO_BIN_FILE_PUBSYS_SETUP")); const TESTSYS: &[u8] = include_bytes!(env!("CARGO_BIN_FILE_TESTSYS")); @@ -44,6 +45,7 @@ pub(crate) async fn install_tools(tools_dir: impl AsRef) -> Result<()> { write_bin("buildsys", BUILDSYS, &dir, mtime).await?; write_bin("pipesys", PIPESYS, &dir, mtime).await?; + #[cfg(feature = "pubsys")] write_bin("pubsys", PUBSYS, &dir, mtime).await?; write_bin("pubsys-setup", PUBSYS_SETUP, &dir, mtime).await?; write_bin("testsys", TESTSYS, &dir, mtime).await?;