From c024e89231ff92e6d31c615de5d8987128cb70fe Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 3 Oct 2023 09:57:33 -0400 Subject: [PATCH] Remove bindle Signed-off-by: Lann Martin --- .devcontainer/Dockerfile | 3 - .devcontainer/devcontainer.json | 1 - .../actions/spin-ci-dependencies/action.yml | 18 - .github/workflows/build.yml | 2 - .github/workflows/code-coverage.yml | 1 - Cargo.lock | 405 +------------- Cargo.toml | 8 - crates/app/src/lib.rs | 2 - crates/bindle/Cargo.toml | 30 -- crates/bindle/src/error.rs | 45 -- crates/bindle/src/expander.rs | 494 ------------------ crates/bindle/src/lib.rs | 12 - crates/bindle/src/pusher.rs | 21 - crates/bindle/src/writer.rs | 176 ------- crates/http/src/app_info.rs | 6 +- crates/loader/Cargo.toml | 4 +- crates/loader/src/bindle/assets.rs | 130 ----- crates/loader/src/bindle/config.rs | 58 -- crates/loader/src/bindle/connection.rs | 86 --- crates/loader/src/bindle/deprecation.rs | 13 - crates/loader/src/bindle/mod.rs | 171 ------ crates/loader/src/bindle/utils.rs | 205 -------- crates/loader/src/lib.rs | 8 +- crates/loader/src/local/config.rs | 12 - crates/loader/tests/invalid-version.toml | 4 +- crates/manifest/src/lib.rs | 7 - crates/plugins/src/lib.rs | 3 +- crates/trigger/src/locked.rs | 7 - src/commands/up.rs | 4 +- tests/integration.rs | 8 - 30 files changed, 34 insertions(+), 1910 deletions(-) delete mode 100644 crates/bindle/Cargo.toml delete mode 100644 crates/bindle/src/error.rs delete mode 100644 crates/bindle/src/expander.rs delete mode 100644 crates/bindle/src/lib.rs delete mode 100644 crates/bindle/src/pusher.rs delete mode 100644 crates/bindle/src/writer.rs delete mode 100644 crates/loader/src/bindle/assets.rs delete mode 100644 crates/loader/src/bindle/config.rs delete mode 100644 crates/loader/src/bindle/connection.rs delete mode 100644 crates/loader/src/bindle/deprecation.rs delete mode 100644 crates/loader/src/bindle/mod.rs delete mode 100644 crates/loader/src/bindle/utils.rs diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 2f8c2bded..b8b56fb80 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -8,9 +8,6 @@ FROM mcr.microsoft.com/vscode/devcontainers/rust:0-${VARIANT} # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends -ARG BINDLE_URL="https://github.com/deislabs/bindle/releases/download/v0.8.0/bindle-v0.8.0-linux-amd64.tar.gz" -RUN curl -sL "$BINDLE_URL" | tar -xzf - -C /usr/local/bin bindle bindle-server - # Go installation, see https://go.dev/doc/install ARG GO_URL="https://go.dev/dl/go1.20.1.linux-amd64.tar.gz" RUN curl -sL "$GO_URL" | tar -xzf - -C /usr/local diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6ebfde4af..62da6187c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,6 @@ "matklad.rust-analyzer", "tamasfe.even-better-toml", "serayuzgur.crates", - "fermyon.autobindle", "golang.Go", "grain-lang.vscode-grain" ], diff --git a/.github/actions/spin-ci-dependencies/action.yml b/.github/actions/spin-ci-dependencies/action.yml index 61ba3e673..912830ee3 100644 --- a/.github/actions/spin-ci-dependencies/action.yml +++ b/.github/actions/spin-ci-dependencies/action.yml @@ -23,17 +23,6 @@ inputs: default: 'false' type: bool - bindle: - description: 'setup bindle' - required: false - default: 'false' - type: bool - bindle-version: - description: 'Bindle version to setup' - default: 'v0.8.0' - required: false - type: string - nomad: description: 'setup nomad' required: false @@ -98,13 +87,6 @@ runs: with: shared-key: "${{ runner.os }}-full-${{ hashFiles('./Cargo.lock') }}" - ## Install bindle - - name: Install bindle - uses: rajatjindal/setup-actions/bindle@v0.0.1 - if: ${{ inputs.bindle == 'true' }} - with: - version: ${{ inputs.bindle-version }} - ## Install nomad - name: Install nomad uses: endocrimes/setup-nomad@0.1.3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d9e96f076..57f8bddf3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,7 +114,6 @@ jobs: rust: true rust-wasm: true rust-cache: true - bindle: true nomad: true # FIXME: THIS IS THE SHORTEST-TERM OF ALL SHORT-TERM FIXES @@ -153,7 +152,6 @@ jobs: uses: ./.github/actions/spin-ci-dependencies with: rust: true - bindle: true golang: true tinygo: true wasmtime: true diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 41b132507..6e30a20b3 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -43,7 +43,6 @@ jobs: with: rust: true rust-wasm: true - bindle: true nomad: true - name: Install cargo-tarpaulin binary crate diff --git a/Cargo.lock b/Cargo.lock index 6648cdb5a..034ffb3b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,7 +39,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", - "cipher 0.4.4", + "cipher", "cpufeatures", ] @@ -187,19 +187,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "async-compression" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-compression" version = "0.4.1" @@ -329,7 +316,7 @@ dependencies = [ "log", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "thiserror", "time 0.3.20", "url", @@ -378,17 +365,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bcrypt" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f691e63585950d8c1c43644d11bab9073e40f5060dd2822734ae7c3dc69a3a80" -dependencies = [ - "base64 0.13.1", - "blowfish", - "getrandom 0.2.8", -] - [[package]] name = "bincode" version = "1.3.3" @@ -417,48 +393,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "bindle" -version = "0.8.0" -source = "git+https://github.com/fermyon/bindle?tag=v0.8.2#84ea1dd86fab6ffe27235e268f47f632421db8f9" -dependencies = [ - "anyhow", - "async-compression 0.3.15", - "async-trait", - "base64 0.13.1", - "bcrypt", - "bytes", - "dirs 4.0.0", - "ed25519-dalek", - "either", - "futures", - "hyper", - "jsonwebtoken", - "lru 0.7.8", - "mime", - "mime_guess", - "oauth2", - "rand 0.7.3", - "reqwest", - "semver", - "serde", - "serde_cbor", - "serde_json", - "sha2 0.10.6", - "sled", - "tempfile", - "thiserror", - "time 0.3.20", - "tokio", - "tokio-stream", - "tokio-tar", - "tokio-util 0.6.10", - "toml 0.5.11", - "tracing", - "tracing-futures", - "url", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -483,15 +417,6 @@ dependencies = [ "wyz", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -501,17 +426,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blowfish" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3ff3fc1de48c1ac2e3341c4df38b0d1bfb8fdf04632a187c8b75aaa319a7ab" -dependencies = [ - "byteorder", - "cipher 0.3.0", - "opaque-debug", -] - [[package]] name = "bstr" version = "1.4.0" @@ -599,7 +513,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "tar", "tempfile", "thiserror", @@ -887,15 +801,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - [[package]] name = "cipher" version = "0.4.4" @@ -1379,7 +1284,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio", - "parking_lot 0.12.1", + "parking_lot", "signal-hook", "signal-hook-mio", "winapi", @@ -1441,19 +1346,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "curve25519-dalek" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" -dependencies = [ - "byteorder", - "digest 0.9.0", - "rand_core 0.5.1", - "subtle", - "zeroize", -] - [[package]] name = "cxx" version = "1.0.92" @@ -1616,22 +1508,13 @@ dependencies = [ "zeroize", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", "subtle", ] @@ -1705,7 +1588,7 @@ dependencies = [ "serde", "serde_ignored", "serde_json", - "sha2 0.10.6", + "sha2", "strum", "strum_macros", "tar", @@ -1776,29 +1659,6 @@ dependencies = [ "url", ] -[[package]] -name = "ed25519" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" -dependencies = [ - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "zeroize", -] - [[package]] name = "either" version = "1.8.1" @@ -2389,7 +2249,7 @@ checksum = "aed73ef9642f779d609fd19acc332ac1597b978ee87ec11743a68eefaed65bfa" dependencies = [ "libc", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "tempfile", ] @@ -2541,7 +2401,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.6", + "digest", ] [[package]] @@ -2976,20 +2836,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonwebtoken" -version = "8.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" -dependencies = [ - "base64 0.13.1", - "pem 1.1.1", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - [[package]] name = "jwt" version = "0.16.0" @@ -2998,11 +2844,11 @@ checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" dependencies = [ "base64 0.13.1", "crypto-common", - "digest 0.10.6", + "digest", "hmac", "serde", "serde_json", - "sha2 0.10.6", + "sha2", ] [[package]] @@ -3358,15 +3204,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "lru" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" -dependencies = [ - "hashbrown 0.12.3", -] - [[package]] name = "lru" version = "0.9.0" @@ -3431,7 +3268,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest 0.10.6", + "digest", ] [[package]] @@ -3617,7 +3454,7 @@ dependencies = [ "mysql_common", "native-tls", "once_cell", - "pem 2.0.1", + "pem", "percent-encoding", "pin-project", "priority-queue", @@ -3658,7 +3495,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.10.5", - "sha2 0.10.6", + "sha2", "smallvec", "subprocess", "thiserror", @@ -3833,26 +3670,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" -[[package]] -name = "oauth2" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaf26a72311c087f8c5ba617c96fac67a5c04f430e716ac8d8ab2de62e23368" -dependencies = [ - "base64 0.13.1", - "chrono", - "getrandom 0.2.8", - "http", - "rand 0.8.5", - "reqwest", - "serde", - "serde_json", - "serde_path_to_error", - "sha2 0.10.6", - "thiserror", - "url", -] - [[package]] name = "object" version = "0.31.1" @@ -3891,7 +3708,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "thiserror", "tokio", "tokio-util 0.7.7", @@ -3944,12 +3761,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.55" @@ -4092,17 +3903,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -4110,21 +3910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.7", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -4193,10 +3979,10 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ - "digest 0.10.6", + "digest", "hmac", "password-hash", - "sha2 0.10.6", + "sha2", ] [[package]] @@ -4205,15 +3991,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "pem" version = "2.0.1" @@ -4271,7 +4048,7 @@ checksum = "5e3b284b1f13a20dc5ebc90aff59a51b8d7137c221131b52a7260c08cbc1cc80" dependencies = [ "once_cell", "pest", - "sha2 0.10.6", + "sha2", ] [[package]] @@ -4411,7 +4188,7 @@ dependencies = [ "md-5", "memchr", "rand 0.8.5", - "sha2 0.10.6", + "sha2", "stringprep", ] @@ -4804,7 +4581,7 @@ version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" dependencies = [ - "async-compression 0.4.1", + "async-compression", "base64 0.21.3", "bytes", "encoding_rs", @@ -5234,15 +5011,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0969fff533976baadd92e08b1d102c5a3d8a8049eadfd69d4d1e3c5b2ed189" -dependencies = [ - "serde", -] - [[package]] name = "serde_qs" version = "0.8.5" @@ -5292,7 +5060,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.6", + "digest", ] [[package]] @@ -5301,19 +5069,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.6" @@ -5322,7 +5077,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.6", + "digest", ] [[package]] @@ -5394,30 +5149,12 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" - [[package]] name = "similar" version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror", - "time 0.3.20", -] - [[package]] name = "siphasher" version = "0.3.10" @@ -5433,22 +5170,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] - [[package]] name = "slice-group-by" version = "0.3.0" @@ -5530,35 +5251,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "spin-bindle" -version = "2.0.0-pre0" -dependencies = [ - "anyhow", - "bindle", - "dirs 4.0.0", - "dunce", - "futures", - "itertools 0.10.5", - "lazy_static", - "mime_guess", - "regex", - "reqwest", - "semver", - "serde", - "serde_json", - "spin-app", - "spin-common", - "spin-loader", - "spin-manifest", - "tempfile", - "thiserror", - "tokio", - "toml 0.5.11", - "tracing", - "walkdir", -] - [[package]] name = "spin-build" version = "2.0.0-pre0" @@ -5580,7 +5272,6 @@ version = "2.0.0-pre0" dependencies = [ "anyhow", "async-trait", - "bindle", "bytes", "cargo-target-dep", "chrono", @@ -5613,9 +5304,8 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "spin-app", - "spin-bindle", "spin-build", "spin-common", "spin-config", @@ -5653,7 +5343,7 @@ version = "2.0.0-pre0" dependencies = [ "anyhow", "dirs 4.0.0", - "sha2 0.10.6", + "sha2", "tempfile", "tokio", ] @@ -5852,7 +5542,6 @@ version = "2.0.0-pre0" dependencies = [ "anyhow", "async-trait", - "bindle", "bytes", "dirs 4.0.0", "dunce", @@ -6547,7 +6236,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.5.3", @@ -6589,7 +6278,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.1", + "parking_lot", "percent-encoding", "phf", "pin-project-lite", @@ -6632,21 +6321,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tar" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50188549787c32c1c3d9c8c71ad7e003ccf2f102489c5a96e385c84760477f4" -dependencies = [ - "filetime", - "futures-core", - "libc", - "redox_syscall 0.2.16", - "tokio", - "tokio-stream", - "xattr", -] - [[package]] name = "tokio-util" version = "0.6.10" @@ -6793,16 +6467,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "tracing-log" version = "0.1.3" @@ -7336,7 +7000,7 @@ dependencies = [ "rustix 0.38.13", "serde", "serde_derive", - "sha2 0.10.6", + "sha2", "toml 0.5.11", "windows-sys 0.48.0", "zstd", @@ -8254,21 +7918,6 @@ name = "zeroize" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "synstructure", -] [[package]] name = "zip" diff --git a/Cargo.toml b/Cargo.toml index 1288855ef..ebac03b0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ edition = "2021" [dependencies] anyhow = "1.0" async-trait = "0.1" -bindle = { workspace = true } bytes = "1.1" chrono = "0.4" clap = { version = "3.2.24", features = ["derive", "env"] } @@ -48,7 +47,6 @@ serde_json = "1.0.82" sha2 = "0.10.2" terminal = { path = "crates/terminal" } spin-app = { path = "crates/app" } -spin-bindle = { path = "crates/bindle" } spin-build = { path = "crates/build" } spin-common = { path = "crates/common" } spin-config = { path = "crates/config" } @@ -114,12 +112,6 @@ wasi-common-preview1 = { version = "13.0.0", package = "wasi-common" } wasmtime = { version = "13.0.0", features = ["component-model"] } spin-componentize = { git = "https://github.com/fermyon/spin-componentize", rev = "0fa79bfc60f20c172213e2a2c34bd0b3168960b0" } -[workspace.dependencies.bindle] -git = "https://github.com/fermyon/bindle" -tag = "v0.8.2" -default-features = false -features = ["client"] - [[bin]] name = "spin" path = "src/bin/spin.rs" diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 3299ff9f1..e3a8d9b71 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -28,8 +28,6 @@ pub use metadata::MetadataKey; pub const APP_NAME_KEY: MetadataKey = MetadataKey::new("name"); /// MetadataKey for extracting the application version. pub const APP_VERSION_KEY: MetadataKey = MetadataKey::new("version"); -/// MetadataKey for extracting the bindle version. -pub const BINDLE_VERSION_KEY: MetadataKey = MetadataKey::new("bindle_version"); /// MetadataKey for extracting the OCI image digest. pub const OCI_IMAGE_DIGEST_KEY: MetadataKey = MetadataKey::new("oci_image_digest"); diff --git a/crates/bindle/Cargo.toml b/crates/bindle/Cargo.toml deleted file mode 100644 index 16530a4ba..000000000 --- a/crates/bindle/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "spin-bindle" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[dependencies] -anyhow = "1.0" -bindle = { workspace = true } -dirs = "4.0" -dunce = "1.0" -futures = "0.3.14" -itertools = "0.10.3" -lazy_static = "1.4.0" -mime_guess = { version = "2.0" } -regex = "1.5.4" -reqwest = "0.11" -semver = "1.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -spin-app = { path = "../app" } -spin-common = { path = "../common" } -spin-loader = { path = "../loader", features = ["bindle"]} -spin-manifest = { path = "../manifest" } -tempfile = "3.3.0" -thiserror = "1.0.37" -tokio = { version = "1.20", features = ["full"] } -tracing = { workspace = true } -toml = "0.5" -walkdir = "2.3.2" diff --git a/crates/bindle/src/error.rs b/crates/bindle/src/error.rs deleted file mode 100644 index 69d9cdaa3..000000000 --- a/crates/bindle/src/error.rs +++ /dev/null @@ -1,45 +0,0 @@ -/// A specialized Result type for publish operations -pub type PublishResult = std::result::Result; - -/// Describes various errors that can be returned during publishing -#[derive(Debug, thiserror::Error)] -pub enum PublishError { - /// The bindle already exists - #[error("Bindle {0} already exists on the server")] - BindleAlreadyExists(String), - /// Bindle client failure - #[error("Error creating bindle client")] - BindleClient(#[from] bindle::client::ClientError), - /// Malformed bindle id - #[error("App name and version '{0}' do not form a bindle ID")] - BindleId(String), - /// Malformed bindle id - #[error("App name '{0}' contains characters not allowed in a bindle name. A bindle name may contain only letters, numbers, and underscores")] - BindleNameInvalidChars(String), - /// Publishing of components whose sources are already bindles is not supported - #[error("This version of Spin can't publish components whose sources are already bindles")] - BindlePushingNotImplemented, - /// IO errors from interacting with the file system - #[error("{description}")] - Io { - /// Error description - description: String, - /// Underlying lower level error that caused your error - source: std::io::Error, - }, - /// Build artifact is missing - #[error("Missing build artifact: '{0}'")] - MissingBuildArtifact(String), - /// Invalid TOML serialization that can occur when serializing an object to a request - #[error("{description}")] - TomlSerialization { - /// Error description - description: String, - /// Underlying lower level error that caused your error - source: toml::ser::Error, - }, - /// A catch-all for anyhow errors. - /// Contains an error message describing the underlying issue. - #[error(transparent)] - Other(#[from] anyhow::Error), -} diff --git a/crates/bindle/src/expander.rs b/crates/bindle/src/expander.rs deleted file mode 100644 index f35ae61cf..000000000 --- a/crates/bindle/src/expander.rs +++ /dev/null @@ -1,494 +0,0 @@ -use super::{ - writer::{self, ParcelSources}, - PublishError, PublishResult, -}; -use bindle::{BindleSpec, Condition, Group, Invoice, Label, Parcel}; -use semver::BuildMetadata; -use spin_common::sha256; -use spin_loader::{ - bindle::config as bindle_schema, - local::{absolutize, config as local_schema, parent_dir, validate_raw_app_manifest, UrlSource}, -}; -use std::path::{Path, PathBuf}; - -/// Expands a file-based application manifest to a Bindle invoice. -pub async fn expand_manifest( - app_file: impl AsRef, - buildinfo: Option, - scratch_dir: impl AsRef, -) -> PublishResult<(Invoice, ParcelSources)> { - let app_file = absolutize(app_file)?; - let manifest = spin_loader::local::raw_manifest_from_file(&app_file).await?; - validate_raw_app_manifest(&manifest)?; - let manifest = manifest.into_v1(); - let app_dir = parent_dir(&app_file)?; - - // * create a new spin.toml-like document where - // - each component changes its `files` entry to a group name - // - each component changes its `source` entry to a parcel SHA - let dest_manifest = bindle_manifest(&manifest, &app_dir).await?; - - // * create an invoice where - // - the metadata is copied from the app manifest - // - there is a group for each component - // - there is a parcel for each asset - // - there is a parcel for each module source - // - if a component refers to an asset then the asset is in the component's group - // - the source and manifest parcels should NOT be group members - // - there is a parcel for the spin.toml-a-like and it has the magic media type - - // - n parcels for the Wasm modules at their locations - let wasm_parcels = wasm_parcels(&manifest, &app_dir, &scratch_dir).await?; - let wasm_parcels = consolidate_wasm_parcels(wasm_parcels); - // - n parcels for the assets under the base directory - let asset_parcels = asset_parcels(&manifest, &app_dir).await?; - let asset_parcels = consolidate_asset_parcels(asset_parcels); - // - one parcel to rule them all, and in the Spin app bind them - let manifest_parcel = manifest_parcel(&dest_manifest, &scratch_dir).await?; - - let sourced_parcels = itertools::concat([vec![manifest_parcel], wasm_parcels, asset_parcels]); - let (parcels, sources) = split_sources(sourced_parcels); - - let bindle_id = bindle_id(&manifest.info, buildinfo)?; - let groups = build_groups(&manifest); - - let invoice = Invoice { - bindle_version: "1.0.0".to_owned(), - yanked: None, - bindle: BindleSpec { - id: bindle_id, - description: manifest.info.description.clone(), - authors: manifest.info.authors.clone(), - }, - annotations: None, - parcel: Some(parcels), - group: Some(groups), - signature: None, - yanked_signature: None, - }; - - Ok((invoice, sources)) -} - -async fn bindle_manifest( - local: &local_schema::RawAppManifest, - base_dir: &Path, -) -> PublishResult { - let futures = local - .components - .iter() - .map(|c| async { bindle_component_manifest(c, base_dir).await }); - let components = futures::future::join_all(futures) - .await - .into_iter() - .collect::>>()?; - let trigger = local.info.trigger.clone(); - let variables = local.variables.clone(); - - Ok(bindle_schema::RawAppManifest { - trigger, - components, - variables, - }) -} - -async fn bindle_component_manifest( - local: &local_schema::RawComponentManifest, - base_dir: &Path, -) -> PublishResult { - let source_digest = match &local.source { - local_schema::RawModuleSource::FileReference(path) => { - let full_path = base_dir.join(path); - - if let Ok(false) = Path::try_exists(&full_path) { - return Err(PublishError::MissingBuildArtifact( - full_path.display().to_string(), - )); - } - - sha256_digest(&full_path)? - } - local_schema::RawModuleSource::Url(us) => { - let source = UrlSource::new(us)?; - source.digest_str().to_owned() - } - }; - let asset_group = local.wasm.files.as_ref().map(|_| group_name_for(&local.id)); - Ok(bindle_schema::RawComponentManifest { - id: local.id.clone(), - description: local.description.clone(), - source: source_digest, - wasm: bindle_schema::RawWasmConfig { - environment: local.wasm.environment.clone(), - files: asset_group, - allowed_http_hosts: local.wasm.allowed_http_hosts.clone(), - key_value_stores: local.wasm.key_value_stores.clone(), - sqlite_databases: local.wasm.sqlite_databases.clone(), - ai_models: local.wasm.ai_models.clone(), - }, - trigger: local.trigger.clone(), - config: local.config.clone(), - }) -} - -async fn wasm_parcels( - manifest: &local_schema::RawAppManifest, - base_dir: &Path, - scratch_dir: impl AsRef, -) -> PublishResult> { - let parcel_futures = manifest - .components - .iter() - .map(|c| wasm_parcel(c, base_dir, scratch_dir.as_ref())); - let parcels = futures::future::join_all(parcel_futures).await; - parcels.into_iter().collect() -} - -async fn wasm_parcel( - component: &local_schema::RawComponentManifest, - base_dir: &Path, - scratch_dir: impl AsRef, -) -> PublishResult { - let (wasm_file, absolute_wasm_file) = match &component.source { - local_schema::RawModuleSource::FileReference(path) => { - (path.to_owned(), base_dir.join(path)) - } - local_schema::RawModuleSource::Url(us) => { - let source = UrlSource::new(us)?; - let bytes = source.get().await?; - let temp_dir = scratch_dir.as_ref().join("downloads"); - let absolute_path = write_file(&temp_dir, &us.digest.replace(':', "_"), &bytes).await?; - let dest_relative_path = source.url_relative_path(); - - (dest_relative_path, absolute_path) - } - }; - - file_parcel(&absolute_wasm_file, wasm_file, None, "application/wasm").await -} - -async fn asset_parcels( - manifest: &local_schema::RawAppManifest, - base_dir: impl AsRef, -) -> PublishResult> { - let assets_by_component: Vec> = manifest - .components - .iter() - .map(|c| collect_assets(c, &base_dir)) - .collect::>()?; - let parcel_futures = assets_by_component - .iter() - .flatten() - .map(|(fm, s)| file_parcel_from_mount(fm, s)); - let parcel_results = futures::future::join_all(parcel_futures).await; - let parcels = parcel_results.into_iter().collect::>()?; - Ok(parcels) -} - -fn collect_assets( - component: &local_schema::RawComponentManifest, - base_dir: impl AsRef, -) -> PublishResult> { - let patterns = component.wasm.files.clone().unwrap_or_default(); - let exclude_files = component.wasm.exclude_files.clone().unwrap_or_default(); - let file_mounts = spin_loader::local::assets::collect(&patterns, &exclude_files, &base_dir)?; - let annotated = file_mounts - .into_iter() - .map(|v| (v, component.id.clone())) - .collect(); - Ok(annotated) -} - -async fn file_parcel_from_mount( - file_mount: &spin_loader::local::assets::FileMount, - component_id: &str, -) -> PublishResult { - let source_file = &file_mount.src; - - let media_type = mime_guess::from_path(source_file) - .first_or_octet_stream() - .to_string(); - - file_parcel( - source_file, - &file_mount.relative_dst, - Some(component_id), - &media_type, - ) - .await -} - -async fn file_parcel( - abs_src: &Path, - dest_relative_path: impl AsRef, - component_id: Option<&str>, - media_type: impl Into, -) -> PublishResult { - let parcel = Parcel { - label: Label { - sha256: sha256_digest(abs_src)?, - name: dest_relative_path.as_ref().display().to_string(), - size: file_metadata(&abs_src).await?.len(), - media_type: media_type.into(), - annotations: None, - feature: None, - origin: None, - }, - conditions: Some(Condition { - member_of: component_id.map(|id| vec![group_name_for(id)]), - requires: None, - }), - }; - - Ok(SourcedParcel { - parcel, - source: abs_src.to_owned(), - }) -} - -async fn manifest_parcel( - manifest: &bindle_schema::RawAppManifest, - scratch_dir: impl AsRef, -) -> PublishResult { - let text = toml::to_string_pretty(&manifest).map_err(|e| PublishError::TomlSerialization { - source: e, - description: "App manifest serialization failure".to_string(), - })?; - let bytes = text.as_bytes(); - let digest = sha256::hex_digest_from_bytes(bytes); - let parcel_name = format!("spin.{}.toml", digest); - let temp_dir = scratch_dir.as_ref().join("manifests"); - let absolute_path = write_file(&temp_dir, &parcel_name, bytes).await?; - - let parcel = Parcel { - label: Label { - sha256: digest.clone(), - name: parcel_name, - size: file_metadata(&absolute_path).await?.len(), - media_type: spin_loader::bindle::SPIN_MANIFEST_MEDIA_TYPE.to_owned(), - annotations: Some(writer::delete_after_copy()), - feature: None, - origin: None, - }, - conditions: None, - }; - - Ok(SourcedParcel { - parcel, - source: absolute_path, - }) -} - -fn consolidate_wasm_parcels(parcels: Vec) -> Vec { - // We use only the content of Wasm parcels, not their names, so we only - // care if the content is the same. - let mut parcels = parcels; - parcels.dedup_by_key(|p| p.parcel.label.sha256.clone()); - parcels -} - -fn consolidate_asset_parcels(parcels: Vec) -> Vec { - let mut consolidated = vec![]; - - for mut parcel in parcels { - match consolidated - .iter_mut() - .find(|p: &&mut SourcedParcel| can_consolidate_asset_parcels(&p.parcel, &parcel.parcel)) - { - None => consolidated.push(parcel), - Some(existing) => { - // If can_consolidate returned true, both parcels must have conditions - // and both conditions must have a member_of list. So these unwraps - // are safe. - // - // TODO: modify can_consolidate to return suitable stuff so we don't - // have to unwrap. - let existing_conds = existing.parcel.conditions.as_mut().unwrap(); - let conds_to_merge = parcel.parcel.conditions.as_mut().unwrap(); - let existing_member_of = existing_conds.member_of.as_mut().unwrap(); - let member_of_to_merge = conds_to_merge.member_of.as_mut().unwrap(); - existing_member_of.append(member_of_to_merge); - } - } - } - - consolidated -} - -fn can_consolidate_asset_parcels(first: &Parcel, second: &Parcel) -> bool { - // For asset parcels, we care not only about the content, but where they - // are placed and whether they have any metadata. For example, if the same - // image is needed both at /resources/logo.png and at /images/header.png, - // we don't want to consolidate those references. - if first.label.name == second.label.name - && first.label.sha256 == second.label.sha256 - && first.label.size == second.label.size - && first.label.media_type == second.label.media_type - && first.label.annotations.is_none() - && second.label.annotations.is_none() - && first.label.feature.is_none() - && second.label.feature.is_none() - { - match (&first.conditions, &second.conditions) { - (Some(c1), Some(c2)) => { - c1.member_of.is_some() - && c2.member_of.is_some() - && c1.requires.is_none() - && c2.requires.is_none() - } - _ => false, - } - } else { - false - } -} - -fn build_groups(manifest: &local_schema::RawAppManifest) -> Vec { - manifest - .components - .iter() - .map(|c| group_for(&c.id)) - .collect() -} - -fn group_name_for(component_id: &str) -> String { - format!("files-{}", component_id) -} - -fn group_for(component_id: &str) -> Group { - Group { - name: group_name_for(component_id), - required: None, - satisfied_by: None, - } -} - -fn bindle_id( - app_info: &local_schema::RawAppInformation, - buildinfo: Option, -) -> PublishResult { - check_safe_bindle_name(&app_info.name)?; - let text = match buildinfo { - None => format!("{}/{}", app_info.name, app_info.version), - Some(buildinfo) => format!("{}/{}+{}", app_info.name, app_info.version, buildinfo), - }; - bindle::Id::try_from(&text).map_err(|_| PublishError::BindleId(text)) -} - -// This is both slightly conservative and slightly loose. According to the spec, -// the / character should also be allowed, but currently that is not supported -// by Hippo (Spin issue 504). And the - character should not be allowed, but lots of -// our tests use it...! -lazy_static::lazy_static! { - static ref SAFE_BINDLE_NAME: regex::Regex = regex::Regex::new("^[-_\\p{L}\\p{N}]+$").expect("Invalid name regex"); -} - -fn check_safe_bindle_name(name: &str) -> PublishResult<()> { - if SAFE_BINDLE_NAME.is_match(name) { - Ok(()) - } else { - Err(PublishError::BindleNameInvalidChars(name.to_owned())) - } -} - -struct SourcedParcel { - parcel: Parcel, - source: PathBuf, -} - -fn split_sources(sourced_parcels: Vec) -> (Vec, ParcelSources) { - let sources = sourced_parcels - .iter() - .map(|sp| (sp.parcel.label.sha256.clone(), &sp.source)); - let parcel_sources = ParcelSources::from_iter(sources); - let parcels = sourced_parcels.into_iter().map(|sp| sp.parcel); - - (parcels.collect(), parcel_sources) -} - -fn sha256_digest(file: impl AsRef) -> PublishResult { - sha256::hex_digest_from_file(&file).map_err(|e| PublishError::Io { - source: e, - description: format!( - "Failed to calculate digest for '{}'", - file.as_ref().display() - ), - }) -} - -async fn write_file(dir: &PathBuf, filename: &String, data: &[u8]) -> PublishResult { - let file = dir.join(filename); - - tokio::fs::create_dir_all(&dir) - .await - .map_err(|e| PublishError::Io { - source: e, - description: format!("Failed to create directory: '{}'", dir.display()), - })?; - - tokio::fs::write(&file, &data) - .await - .map_err(|e| PublishError::Io { - source: e, - description: format!("Failed to write file: '{}'", file.display()), - })?; - - dunce::canonicalize(&file).map_err(|e| PublishError::Io { - source: e, - description: format!("Failed to get absolute path: '{}'", file.display()), - }) -} - -async fn file_metadata(file: impl AsRef) -> PublishResult { - tokio::fs::metadata(&file) - .await - .map_err(|e| PublishError::Io { - source: e, - description: format!("Failed to get file metadata: '{}'", file.as_ref().display()), - }) -} - -#[cfg(test)] -mod test { - use spin_loader::local::config::RawAppInformation; - - use super::*; - - fn app_info(name: &str) -> RawAppInformation { - app_info_v(name, "0.0.1") - } - - fn app_info_v(name: &str, version: &str) -> RawAppInformation { - RawAppInformation { - name: name.to_owned(), - version: version.to_owned(), - description: None, - authors: None, - trigger: spin_manifest::ApplicationTrigger::Http( - spin_manifest::HttpTriggerConfiguration { - base: "/".to_owned(), - }, - ), - namespace: None, - } - } - - #[test] - fn accepts_only_valid_bindle_names() { - bindle_id(&app_info("hello"), None).expect("should have accepted 'hello'"); - bindle_id(&app_info("hello-world"), None).expect("should have accepted 'hello-world'"); - bindle_id(&app_info("hello_world"), None).expect("should have accepted 'hello_world'"); - - let err = bindle_id(&app_info("hello/world"), None) - .expect_err("should not have accepted 'hello/world'"); - assert!(matches!(err, PublishError::BindleNameInvalidChars(_))); - - let err = bindle_id(&app_info("hello world"), None) - .expect_err("should not have accepted 'hello world'"); - assert!(matches!(err, PublishError::BindleNameInvalidChars(_))); - - let err = bindle_id(&app_info_v("hello", "lolsnort"), None) - .expect_err("should not have accepted version 'lolsnort'"); - assert!(matches!(err, PublishError::BindleId(_))); - } -} diff --git a/crates/bindle/src/lib.rs b/crates/bindle/src/lib.rs deleted file mode 100644 index 7bd521f2e..000000000 --- a/crates/bindle/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Functions for publishing Spin applications to Bindle. -#![deny(missing_docs)] - -mod error; -mod expander; -mod pusher; -mod writer; - -pub use error::{PublishError, PublishResult}; -pub use expander::expand_manifest; -pub use pusher::push_all; -pub use writer::{prepare_bindle, write}; diff --git a/crates/bindle/src/pusher.rs b/crates/bindle/src/pusher.rs deleted file mode 100644 index d05136147..000000000 --- a/crates/bindle/src/pusher.rs +++ /dev/null @@ -1,21 +0,0 @@ -use super::{PublishError, PublishResult}; -use bindle::{standalone::StandaloneRead, Id}; -use std::path::Path; - -/// Pushes a standalone bindle to a Bindle server. -pub async fn push_all( - path: impl AsRef, - bindle_id: &Id, - bindle_connection_info: spin_loader::bindle::BindleConnectionInfo, -) -> PublishResult<()> { - let reader = StandaloneRead::new(&path, bindle_id).await?; - let client = &bindle_connection_info.client()?; - - if client.get_yanked_invoice(bindle_id).await.is_ok() { - return Err(PublishError::BindleAlreadyExists(bindle_id.to_string())); - } - - reader.push(client).await?; - - Ok(()) -} diff --git a/crates/bindle/src/writer.rs b/crates/bindle/src/writer.rs deleted file mode 100644 index e90a3ca58..000000000 --- a/crates/bindle/src/writer.rs +++ /dev/null @@ -1,176 +0,0 @@ -use super::{expander::expand_manifest, PublishError, PublishResult}; -use bindle::{Invoice, Parcel}; -use spin_loader::local::parent_dir; -use std::{ - collections::BTreeMap, - path::{Path, PathBuf}, -}; - -/// Expands a file-based application manifest to a Bindle invoice and writes it -/// as a standalone bindle. -pub async fn prepare_bindle( - app_file: impl AsRef, - buildinfo: Option, - dest_dir: impl AsRef, -) -> PublishResult { - let (invoice, sources) = expand_manifest(&app_file, buildinfo, &dest_dir).await?; - let source_dir = parent_dir(&app_file)?; - - write(&source_dir, &dest_dir, &invoice, &sources).await?; - - Ok(invoice.bindle.id) -} - -struct BindleWriter { - source_dir: PathBuf, - dest_dir: PathBuf, - invoice: Invoice, - parcel_sources: ParcelSources, -} - -/// Writes an invoice and supporting parcels out as a standalone bindle. -pub async fn write( - source_dir: impl AsRef, - dest_dir: impl AsRef, - invoice: &Invoice, - parcel_sources: &ParcelSources, -) -> PublishResult<()> { - let writer = BindleWriter { - source_dir: source_dir.as_ref().to_owned(), - dest_dir: dest_dir.as_ref().to_owned(), - invoice: invoice.clone(), - parcel_sources: parcel_sources.clone(), - }; - writer.write().await -} - -impl BindleWriter { - async fn write(&self) -> PublishResult<()> { - // This is very similar to bindle::StandaloneWrite::write but... not quite the same - let bindle_id_hash = self.invoice.bindle.id.sha(); - let bindle_dir = self.dest_dir.join(bindle_id_hash); - let parcels_dir = bindle_dir.join("parcels"); - tokio::fs::create_dir_all(&parcels_dir) - .await - .map_err(|e| PublishError::Io { - source: e, - description: format!("Failed to create parcels dir: {}", parcels_dir.display()), - })?; - - self.write_invoice_file(&bindle_dir).await?; - self.write_parcel_files(&parcels_dir).await?; - Ok(()) - } - - async fn write_invoice_file(&self, bindle_dir: &Path) -> PublishResult<()> { - let invoice_text = - toml::to_string_pretty(&self.invoice).map_err(|e| PublishError::TomlSerialization { - source: e, - description: format!("Failed to serialize invoice: {:?}", self.invoice), - })?; - let invoice_file = bindle_dir.join("invoice.toml"); - - tokio::fs::write(&invoice_file, &invoice_text) - .await - .map_err(|e| PublishError::Io { - source: e, - description: format!("Failed to write invoice to '{}'", invoice_file.display()), - })?; - Ok(()) - } - - async fn write_parcel_files(&self, parcels_dir: &Path) -> PublishResult<()> { - let parcels = match &self.invoice.parcel { - Some(p) => p, - None => return Ok(()), - }; - - let parcel_writes = parcels - .iter() - .map(|parcel| self.write_one_parcel(parcels_dir, parcel)); - - futures::future::join_all(parcel_writes) - .await - .into_iter() - .collect::>>()?; - - Ok(()) - } - - async fn write_one_parcel(&self, parcels_dir: &Path, parcel: &Parcel) -> PublishResult<()> { - let source_file = match self.parcel_sources.source(&parcel.label.sha256) { - Some(path) => path.clone(), - None => self.source_dir.join(&parcel.label.name), - }; - let hash = &parcel.label.sha256; - let dest_file = parcels_dir.join(format!("{}.dat", hash)); - tokio::fs::copy(&source_file, &dest_file) - .await - .map_err(|e| PublishError::Io { - description: format!( - "Failed to copy parcel from {} to '{}'", - source_file.display(), - dest_file.display() - ), - source: e, - })?; - - if has_annotation(parcel, DELETE_ON_WRITE) { - tokio::fs::remove_file(&source_file).await.ignore_errors(); // Leaking a temp file is sad but not a reason to fail - } - - Ok(()) - } -} - -#[derive(Debug, Clone)] -pub(crate) struct ParcelSource { - digest: String, - source_path: PathBuf, -} - -#[derive(Debug, Clone)] -pub struct ParcelSources { - sources: Vec, -} - -impl ParcelSources { - pub fn source(&self, digest: &str) -> Option<&PathBuf> { - self.sources - .iter() - .find(|s| s.digest == digest) - .map(|s| &s.source_path) - } - - pub(crate) fn from_iter(paths: impl Iterator)>) -> Self { - let sources = paths - .map(|(digest, path)| ParcelSource { - digest, - source_path: path.as_ref().to_owned(), - }) - .collect(); - - Self { sources } - } -} - -fn has_annotation(parcel: &Parcel, key: &str) -> bool { - match &parcel.label.annotations { - Some(map) => map.contains_key(key), - None => false, - } -} - -const DELETE_ON_WRITE: &str = "fermyon:spin:delete_on_write"; - -pub(crate) fn delete_after_copy() -> BTreeMap { - BTreeMap::from([(DELETE_ON_WRITE.to_owned(), ".".to_owned())]) -} - -trait IgnoreErrors { - fn ignore_errors(&self); -} - -impl IgnoreErrors for std::io::Result<()> { - fn ignore_errors(&self) {} -} diff --git a/crates/http/src/app_info.rs b/crates/http/src/app_info.rs index 658b2fbc9..2b88ec3ed 100644 --- a/crates/http/src/app_info.rs +++ b/crates/http/src/app_info.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use spin_app::{App, APP_NAME_KEY, APP_VERSION_KEY, BINDLE_VERSION_KEY, OCI_IMAGE_DIGEST_KEY}; +use spin_app::{App, APP_NAME_KEY, APP_VERSION_KEY, OCI_IMAGE_DIGEST_KEY}; #[derive(Debug, Serialize, Deserialize)] pub struct AppInfo { @@ -7,8 +7,6 @@ pub struct AppInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub bindle_version: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub oci_image_digest: Option, } @@ -19,12 +17,10 @@ impl AppInfo { .unwrap_or_default() .unwrap_or_default(); let version = app.get_metadata(APP_VERSION_KEY).unwrap_or_default(); - let bindle_version = app.get_metadata(BINDLE_VERSION_KEY).unwrap_or_default(); let oci_image_digest = app.get_metadata(OCI_IMAGE_DIGEST_KEY).unwrap_or_default(); Self { name, version, - bindle_version, oci_image_digest, } } diff --git a/crates/loader/Cargo.toml b/crates/loader/Cargo.toml index 84f66ae9b..fe71337eb 100644 --- a/crates/loader/Cargo.toml +++ b/crates/loader/Cargo.toml @@ -7,7 +7,6 @@ edition = { workspace = true } [dependencies] anyhow = "1" async-trait = "0.1.52" -bindle = { workspace = true, optional = true } bytes = "1.1.0" dirs = "4.0" dunce = "1.0" @@ -36,6 +35,5 @@ tracing = { workspace = true } walkdir = "2.3.2" [features] -default = ["bindle", "local"] -bindle = ["dep:bindle"] +default = ["local"] local = [] \ No newline at end of file diff --git a/crates/loader/src/bindle/assets.rs b/crates/loader/src/bindle/assets.rs deleted file mode 100644 index df35f00a3..000000000 --- a/crates/loader/src/bindle/assets.rs +++ /dev/null @@ -1,130 +0,0 @@ -#![deny(missing_docs)] - -use std::path::Path; -use std::path::PathBuf; - -use anyhow::{anyhow, bail, Context, Result}; -use bindle::{Id, Label}; -use futures::{future, stream, StreamExt, TryStreamExt}; -use spin_common::sha256; -use spin_manifest::DirectoryMount; -use tokio::{fs, io::AsyncWriteExt}; -use tracing::log; - -use crate::{ - assets::{create_dir, ensure_under}, - bindle::utils::BindleReader, - local::parent_dir, -}; - -pub(crate) async fn prepare_component( - reader: &BindleReader, - bindle_id: &Id, - parcels: &[Label], - base_dst: impl AsRef, - component: &str, -) -> Result { - let copier = Copier { - reader: reader.clone(), - id: bindle_id.clone(), - }; - copier.prepare(parcels, base_dst, component).await -} - -pub(crate) struct Copier { - reader: BindleReader, - id: Id, -} - -impl Copier { - async fn prepare( - &self, - parcels: &[Label], - base_dst: impl AsRef, - component: &str, - ) -> Result { - log::info!( - "Mounting files from '{}' to '{}'", - self.id, - base_dst.as_ref().display() - ); - - let host = create_dir(&base_dst, component).await?; - let guest = "/".to_string(); - self.copy_all(parcels, &host).await?; - - Ok(DirectoryMount { host, guest }) - } - - async fn copy_all(&self, parcels: &[Label], dir: impl AsRef) -> Result<()> { - match stream::iter(parcels.iter().map(|p| self.copy(p, &dir))) - .buffer_unordered(crate::MAX_PARALLEL_ASSET_PROCESSING) - .filter_map(|r| future::ready(r.err())) - .map(|e| log::error!("{:?}", e)) - .count() - .await - { - 0 => Ok(()), - n => bail!("Error copying assets: {} file(s) not copied", n), - } - } - - async fn copy(&self, p: &Label, dir: impl AsRef) -> Result<()> { - let to = dir.as_ref().join(&p.name); - - ensure_under(&dir, &to)?; - - if to.exists() { - match check_existing_file(to.clone(), p).await { - // Copy already exists - Ok(true) => return Ok(()), - Ok(false) => (), - Err(err) => tracing::error!("Error verifying existing parcel: {}", err), - } - } - - log::trace!( - "Copying asset file '{}@{}' -> '{}'", - self.id, - p.sha256, - to.display() - ); - fs::create_dir_all(parent_dir(&to).expect("Cannot copy to file '/'")).await?; - let mut stream = self - .reader - .get_parcel_stream(&p.sha256) - .await - .with_context(|| anyhow!("Failed to fetch asset parcel '{}@{}'", self.id, p.sha256))?; - - let mut file = fs::File::create(&to).await.with_context(|| { - anyhow!( - "Failed to create local file for asset parcel '{}@{}'", - self.id, - p.sha256 - ) - })?; - - while let Some(chunk) = stream - .try_next() - .await - .with_context(|| anyhow!("Failed to read asset parcel '{}@{}'", self.id, p.sha256))? - { - file.write_all(&chunk).await.with_context(|| { - anyhow!( - "Failed to write asset parcel '{}@{}' to {}", - self.id, - p.sha256, - to.display() - ) - })?; - } - - Ok(()) - } -} - -async fn check_existing_file(path: PathBuf, label: &Label) -> Result { - let sha256_digest = - tokio::task::spawn_blocking(move || sha256::hex_digest_from_file(path)).await??; - Ok(sha256_digest == label.sha256) -} diff --git a/crates/loader/src/bindle/config.rs b/crates/loader/src/bindle/config.rs deleted file mode 100644 index 3792d300c..000000000 --- a/crates/loader/src/bindle/config.rs +++ /dev/null @@ -1,58 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -use crate::common::RawVariable; - -/// Application configuration file format. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct RawAppManifest { - /// The application trigger. - pub trigger: spin_manifest::ApplicationTrigger, - - /// Application-specific configuration schema. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub variables: HashMap, - - /// Configuration for the application components. - #[serde(rename = "component")] - pub components: Vec, -} - -/// Core component configuration. -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct RawComponentManifest { - /// The module source. - pub source: String, - /// ID of the component. Used at runtime to select between - /// multiple components of the same application. - pub id: String, - /// Description of the component. - pub description: Option, - /// Per-component WebAssembly configuration. - #[serde(flatten)] - pub wasm: RawWasmConfig, - /// Trigger configuration. - pub trigger: spin_manifest::TriggerConfig, - /// Component-specific configuration values. - pub config: Option>, -} - -/// WebAssembly configuration. -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct RawWasmConfig { - /// The parcel group to be mapped inside the Wasm module at runtime. - pub files: Option, - /// Optional list of HTTP hosts the component is allowed to connect. - pub allowed_http_hosts: Option>, - /// Optional list of key-value stores the component is allowed to use. - pub key_value_stores: Option>, - /// Optional list of SQLite databases the component is allowed to use. - pub sqlite_databases: Option>, - /// Optional list of ai models the component is allowed to use. - pub ai_models: Option>, - /// Environment variables to be mapped inside the Wasm module at runtime. - pub environment: Option>, -} diff --git a/crates/loader/src/bindle/connection.rs b/crates/loader/src/bindle/connection.rs deleted file mode 100644 index 456255257..000000000 --- a/crates/loader/src/bindle/connection.rs +++ /dev/null @@ -1,86 +0,0 @@ -use std::sync::Arc; - -use bindle::client::{ - tokens::{HttpBasic, LongLivedToken, NoToken, TokenManager}, - Client, ClientBuilder, -}; - -/// BindleConnectionInfo holds the details of a connection to a -/// Bindle server, including url, insecure configuration and an -/// auth token manager -#[derive(Clone)] -pub struct BindleConnectionInfo { - base_url: String, - allow_insecure: bool, - token_manager: AnyAuth, -} - -impl BindleConnectionInfo { - /// Generates a new BindleConnectionInfo instance using the provided - /// base_url, allow_insecure setting and optional username and password - /// for basic http auth - pub fn new>( - base_url: I, - allow_insecure: bool, - username: Option, - password: Option, - ) -> Self { - let token_manager: Box = match (username, password) { - (Some(u), Some(p)) => Box::new(HttpBasic::new(&u, &p)), - _ => Box::::default(), - }; - - Self { - base_url: base_url.into(), - allow_insecure, - token_manager: AnyAuth { - token_manager: Arc::new(token_manager), - }, - } - } - - /// Generates a new BindleConnectionInfo instance using the provided - /// base_url, allow_insecure setting and token. - pub fn from_token>(base_url: I, allow_insecure: bool, token: I) -> Self { - let token_manager: Box = - Box::new(LongLivedToken::new(&token.into())); - - Self { - base_url: base_url.into(), - allow_insecure, - token_manager: AnyAuth { - token_manager: Arc::new(token_manager), - }, - } - } - - /// Returns a client based on this instance's configuration - pub fn client(&self) -> bindle::client::Result> { - let builder = ClientBuilder::default() - .http2_prior_knowledge(false) - .danger_accept_invalid_certs(self.allow_insecure); - builder.build(&self.base_url, self.token_manager.clone()) - } - - /// Returns the base url for this client. - pub fn base_url(&self) -> &str { - self.base_url.as_ref() - } -} - -/// AnyAuth wraps an authentication token manager which applies -/// the appropriate auth header per its configuration -#[derive(Clone)] -pub struct AnyAuth { - token_manager: Arc>, -} - -#[async_trait::async_trait] -impl TokenManager for AnyAuth { - async fn apply_auth_header( - &self, - builder: reqwest::RequestBuilder, - ) -> bindle::client::Result { - self.token_manager.apply_auth_header(builder).await - } -} diff --git a/crates/loader/src/bindle/deprecation.rs b/crates/loader/src/bindle/deprecation.rs deleted file mode 100644 index 23b611b84..000000000 --- a/crates/loader/src/bindle/deprecation.rs +++ /dev/null @@ -1,13 +0,0 @@ -use std::sync::Once; - -/// Prints a notice that Bindle support is deprecated. -pub fn print_bindle_deprecation() { - static PRINT: Once = Once::new(); - - PRINT.call_once(|| { - terminal::warn!("Bindle support is deprecated and will be removed in a future version."); - eprintln!("For remote applications, please use registry commands and features instead."); - eprintln!("See https://developer.fermyon.com/spin/spin-oci for more details"); - eprintln!(); - }); -} diff --git a/crates/loader/src/bindle/mod.rs b/crates/loader/src/bindle/mod.rs deleted file mode 100644 index 995d7dc89..000000000 --- a/crates/loader/src/bindle/mod.rs +++ /dev/null @@ -1,171 +0,0 @@ -//! Functionality to get a prepared Spin application from Bindle. - -#![deny(missing_docs)] - -/// Module to prepare the assets for the components of an application. -mod assets; -/// Configuration representation for a Spin application in Bindle. -pub mod config; -mod connection; -/// Functions relating to Bindle deprecation. -pub mod deprecation; -/// Bindle helper functions. -mod utils; - -use std::path::Path; - -use anyhow::{anyhow, Context, Result}; -use bindle::Invoice; -use futures::future; -use outbound_http::allowed_http_hosts::validate_allowed_http_hosts; -use spin_manifest::{ - Application, ApplicationInformation, ApplicationOrigin, CoreComponent, ModuleSource, - SpinVersion, WasmConfig, -}; - -use crate::bindle::{ - config::{RawAppManifest, RawComponentManifest}, - utils::{find_manifest, parcels_in_group}, -}; -pub use connection::BindleConnectionInfo; -pub(crate) use utils::BindleReader; -pub use utils::SPIN_MANIFEST_MEDIA_TYPE; - -/// Given a Bindle server URL and reference, pull it, expand its assets locally, and get a -/// prepared application configuration consumable by a Spin execution context. -/// If a directory is provided, use it as the base directory to expand the assets, -/// otherwise create a new temporary directory. -pub async fn from_bindle(id: &str, url: &str, base_dst: impl AsRef) -> Result { - // TODO - // Handle Bindle authentication. - let connection_info = BindleConnectionInfo::new(url, false, None, None); - let client = connection_info.client()?; - let reader = BindleReader::remote(&client, &id.parse()?); - - prepare(id, url, &reader, base_dst).await -} - -/// Converts a Bindle invoice into Spin configuration. -async fn prepare( - id: &str, - url: &str, - reader: &BindleReader, - base_dst: impl AsRef, -) -> Result { - // First, get the invoice from the Bindle server. - let invoice = reader - .get_invoice() - .await - .with_context(|| anyhow!("Failed to load invoice '{}' from '{}'", id, url))?; - - // Then, reconstruct the application manifest from the parcels. - let raw: RawAppManifest = - toml::from_slice(&reader.get_parcel(&find_manifest(&invoice)?).await?)?; - tracing::trace!("Recreated manifest from bindle: {:?}", raw); - - validate_raw_app_manifest(&raw)?; - - let info = info(&raw, &invoice, url); - tracing::trace!("Application information from bindle: {:?}", info); - let component_triggers = raw - .components - .iter() - .map(|c| (c.id.clone(), c.trigger.clone())) - .collect(); - let components = future::join_all( - raw.components - .into_iter() - .map(|c| async { core(c, &invoice, reader, &base_dst).await }) - .collect::>(), - ) - .await - .into_iter() - .map(|x| x.expect("Cannot prepare component")) - .collect::>(); - - let variables = raw - .variables - .into_iter() - .map(|(key, var)| Ok((key, var.try_into()?))) - .collect::>()?; - - Ok(Application { - info, - variables, - components, - component_triggers, - }) -} - -fn validate_raw_app_manifest(raw: &RawAppManifest) -> Result<()> { - raw.components - .iter() - .try_for_each(|c| validate_allowed_http_hosts(&c.wasm.allowed_http_hosts)) -} - -/// Given a raw component manifest, prepare its assets and return a fully formed core component. -async fn core( - raw: RawComponentManifest, - invoice: &Invoice, - reader: &BindleReader, - base_dst: impl AsRef, -) -> Result { - let bytes = reader - .get_parcel(&raw.source) - .await - .with_context(|| anyhow!("Cannot get module source from bindle"))?; - - let source = ModuleSource::Buffer(bytes, format!("parcel {}", raw.source)); - let id = raw.id; - let description = raw.description; - let mounts = match raw.wasm.files { - Some(group) => { - let parcels = parcels_in_group(invoice, &group); - vec![ - assets::prepare_component(reader, &invoice.bindle.id, &parcels, base_dst, &id) - .await?, - ] - } - None => vec![], - }; - let environment = raw.wasm.environment.unwrap_or_default(); - let allowed_http_hosts = raw.wasm.allowed_http_hosts.unwrap_or_default(); - let key_value_stores = raw.wasm.key_value_stores.unwrap_or_default(); - let sqlite_databases = raw.wasm.sqlite_databases.unwrap_or_default(); - let ai_models = raw.wasm.ai_models.unwrap_or_default(); - let wasm = WasmConfig { - environment, - mounts, - allowed_http_hosts, - key_value_stores, - sqlite_databases, - ai_models, - }; - let config = raw.config.unwrap_or_default(); - Ok(CoreComponent { - source, - id, - description, - wasm, - config, - }) -} - -/// Converts the raw application manifest from the bindle invoice into the -/// standard application configuration. -fn info(raw: &RawAppManifest, invoice: &Invoice, url: &str) -> ApplicationInformation { - ApplicationInformation { - // TODO - // Handle API version - spin_version: SpinVersion::V1, - name: invoice.bindle.id.name().to_string(), - version: invoice.bindle.id.version_string(), - description: invoice.bindle.description.clone(), - authors: invoice.bindle.authors.clone().unwrap_or_default(), - trigger: raw.trigger.clone(), - origin: ApplicationOrigin::Bindle { - id: invoice.bindle.id.to_string(), - server: url.to_string(), - }, - } -} diff --git a/crates/loader/src/bindle/utils.rs b/crates/loader/src/bindle/utils.rs deleted file mode 100644 index eccf142d5..000000000 --- a/crates/loader/src/bindle/utils.rs +++ /dev/null @@ -1,205 +0,0 @@ -#![deny(missing_docs)] - -use anyhow::{anyhow, bail, Context, Error, Result}; -use bindle::{client::Client, standalone::StandaloneRead, Id, Invoice, Label, Parcel}; -use futures::{Stream, StreamExt, TryStreamExt}; -use itertools::Itertools; -use std::{fmt::Debug, path::Path, sync::Arc}; -use tokio::fs; -use tokio_util::codec::{BytesCodec, FramedRead}; - -use super::connection::AnyAuth; - -static EMPTY: &Vec = &vec![]; - -// Alternative to storing `spin.toml` as a parcel, this could be -// distinguished it through a group, or an annotation. - -/// The media type of a `spin.toml` parcel as part of a bindle. -pub const SPIN_MANIFEST_MEDIA_TYPE: &str = "application/vnd.fermyon.spin+toml"; - -pub(crate) fn find_manifest(inv: &Invoice) -> Result { - let parcels = inv - .parcel - .as_ref() - .unwrap_or(EMPTY) - .iter() - .filter_map(|p| { - if p.label.media_type == SPIN_MANIFEST_MEDIA_TYPE { - Some(&p.label) - } else { - None - } - }) - .collect_vec(); - - match parcels.len() { - 0 => bail!("Invoice does not contain a Spin manifest"), - 1 => Ok(parcels[0].sha256.clone()), - _ => bail!("Invoice contains multiple Spin manifests"), - } -} - -// This isn't currently transitive - I don't think we have a need for that -// but could add it if we did (WAGI has code for this but it's a huge faff) -pub(crate) fn parcels_in_group(inv: &Invoice, group: &str) -> Vec