From 038e311f80b33d8e3a67739109ac904134a6025e Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 15 Nov 2023 22:49:43 +0000 Subject: [PATCH 01/19] Add ability to pre-compile modules in wasmtime Signed-off-by: James Sturtevant --- Cargo.lock | 3 + crates/containerd-shim-wasm/Cargo.toml | 3 + .../src/container/context.rs | 4 + .../src/container/engine.rs | 13 +- .../src/sandbox/containerd.rs | 351 ++++++++++++++++-- .../containerd-shim-wasm/src/sandbox/oci.rs | 1 + .../src/sys/unix/container/instance.rs | 2 +- .../containerd-shim-wasmtime/src/instance.rs | 30 ++ 8 files changed, 369 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a18d3e5ce..438fa31ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,12 +628,15 @@ dependencies = [ "log", "nix 0.27.1", "oci-spec", + "prost-types 0.11.9", "protobuf 3.2.0", "serde", "serde_json", + "sha256", "tempfile", "thiserror", "tokio", + "tokio-stream", "ttrpc", "ttrpc-codegen", "wasmparser 0.121.0", diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index 9c30f5c68..ccdd22dee 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -32,6 +32,9 @@ wat = { workspace = true } tokio = { version = "1.35.1", features = [ "full" ] } futures = { version = "0.3.30" } wasmparser = "0.121.0" +tokio-stream = { version = "0.1" } +prost-types = "0.11" # should match version in containerd-shim +sha256 = "1.4.0" [target.'cfg(unix)'.dependencies] caps = "0.5" diff --git a/crates/containerd-shim-wasm/src/container/context.rs b/crates/containerd-shim-wasm/src/container/context.rs index 5f4475a5b..f93a7e66d 100644 --- a/crates/containerd-shim-wasm/src/container/context.rs +++ b/crates/containerd-shim-wasm/src/container/context.rs @@ -45,6 +45,7 @@ pub enum Source<'a> { // and they will be included in this array, e.g., a `toml` file with the // runtime configuration. Oci(&'a [WasmLayer]), + Precompiled(&'a WasmLayer), } impl<'a> Source<'a> { @@ -100,6 +101,8 @@ impl RuntimeContext for WasiContext<'_> { let source = if self.wasm_layers.is_empty() { Source::File(PathBuf::from(path)) + } else if self.wasm_layers.len() == 1 && self.wasm_layers[0].precompiled { + Source::Precompiled(&self.wasm_layers[0]) } else { Source::Oci(self.wasm_layers) }; @@ -358,6 +361,7 @@ mod tests { wasm_layers: &[WasmLayer { layer: vec![], config: Descriptor::new(oci_spec::image::MediaType::Other("".to_string()), 10, ""), + precompiled: false, }], platform: &Platform::default(), }; diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 77f10cc3f..2b9119364 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::Read; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use super::Source; use crate::container::{PathResolve, RuntimeContext}; @@ -26,6 +26,7 @@ pub trait Engine: Clone + Send + Sync + 'static { let path = match source { Source::File(path) => path, Source::Oci(_) => return Ok(()), + Source::Precompiled(_) => return Ok(()), }; path.resolve_in_path_or_cwd() @@ -52,4 +53,14 @@ pub trait Engine: Clone + Send + Sync + 'static { fn supported_layers_types() -> &'static [&'static str] { &["application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"] } + + /// Precomiple a module + fn precompile(&self, _layers: &[Vec]) -> Result> { + bail!("precompilation not supported for this runtime") + } + + /// Precomiple a module + fn can_precompile() -> bool { + false + } } diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 8a5d2782b..190b23490 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -1,19 +1,29 @@ #![cfg(unix)] +use std::collections::HashMap; use std::path::Path; use containerd_client; use containerd_client::services::v1::containers_client::ContainersClient; use containerd_client::services::v1::content_client::ContentClient; use containerd_client::services::v1::images_client::ImagesClient; -use containerd_client::services::v1::{GetContainerRequest, GetImageRequest, ReadContentRequest}; +use containerd_client::services::v1::{ + Container, DeleteContentRequest, GetContainerRequest, GetImageRequest, Image, Info, + InfoRequest, ReadContentRequest, UpdateImageRequest, UpdateRequest, WriteAction, + WriteContentRequest, +}; use containerd_client::tonic::transport::Channel; use containerd_client::{tonic, with_namespace}; use futures::TryStreamExt; use oci_spec::image::{Arch, ImageManifest, MediaType, Platform}; +use prost_types::FieldMask; +use sha256::digest; use tokio::runtime::Runtime; -use tonic::Request; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Code, Request}; +use crate::container::Engine; use crate::sandbox::error::{Error as ShimError, Result}; use crate::sandbox::oci::{self, WasmLayer}; @@ -62,12 +72,162 @@ impl Client { }) } - pub fn get_image_content_sha(&self, image_name: impl ToString) -> Result { + // used in tests to clean up content + #[allow(dead_code)] + fn delete_content(&self, digest: impl ToString) -> Result<()> { + self.rt.block_on(async { + let req = DeleteContentRequest { + digest: digest.to_string(), + }; + let req = with_namespace!(req, self.namespace); + ContentClient::new(self.inner.clone()) + .delete(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))?; + Ok(()) + }) + } + + pub fn save_content(&self, data: Vec) -> Result { + self.rt.block_on(async { + // create a channel to feed the stream; only sending one message at a time so we can set this to one + let (tx, rx) = mpsc::channel(1); + + let len = data.len() as i64; + let expected = digest(data.clone()); + let expected = format!("sha256:{}", expected); + let mut client = ContentClient::new(self.inner.clone()); + let r#ref = "test".to_string(); + + // Send Stat action to containerd to let it know that we are going to write content + // if the content is already there, it will return early with AlreadyExists + let req = WriteContentRequest { + r#ref: r#ref.clone(), + action: WriteAction::Stat.into(), + total: len, + expected: expected.clone(), + ..Default::default() + }; + tx.send(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))?; + let request_stream = ReceiverStream::new(rx); + let request_stream = with_namespace!(request_stream, self.namespace); + let mut response_stream = match client.write(request_stream).await { + Ok(response_stream) => response_stream.into_inner(), + Err(e) if e.code() == Code::AlreadyExists => { + log::info!("content already exists {}", expected.clone().to_string()); + return Ok(expected); + } + Err(e) => return Err(ShimError::Containerd(e.to_string())), + }; + let response = response_stream + .message() + .await + .map_err(|e| ShimError::Containerd(e.to_string()))? + .ok_or_else(|| { + ShimError::Containerd(format!( + "no response received after write request for {}", + expected.clone() + )) + })?; + + // Write and commit at same time + let mut labels = HashMap::new(); + labels.insert("runwasi.io/precompiled".to_string(), "".to_string()); + let commit_request = WriteContentRequest { + action: WriteAction::Commit.into(), + total: len, + offset: response.offset, + expected: expected.clone(), + labels, + data, + ..Default::default() + }; + tx.send(commit_request) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))?; + let response = response_stream + .message() + .await + .map_err(|e| ShimError::Containerd(e.to_string()))? + .ok_or_else(|| { + ShimError::Containerd(format!( + "no response received after write request for {}", + expected.clone() + )) + })?; + + // client should validate that all bytes were written and that the digest matches + if response.offset != len { + return Err(ShimError::Containerd(format!( + "failed to write all bytes, expected {} got {}", + len, response.offset + ))); + } + if response.digest != expected { + return Err(ShimError::Containerd(format!( + "unexpected digest, expected {} got {}", + expected, response.digest + ))); + } + Ok(response.digest) + }) + } + + pub fn get_info(&self, content_digest: String) -> Result { + self.rt.block_on(async { + let req = InfoRequest { + digest: content_digest.clone(), + }; + let req = with_namespace!(req, self.namespace); + let info = ContentClient::new(self.inner.clone()) + .info(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .info + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get info for content {}", + content_digest + )) + })?; + Ok(info) + }) + } + + pub fn update_info(&self, info: Info) -> Result { + self.rt.block_on(async { + let req = UpdateRequest { + info: Some(info.clone()), + update_mask: Some(FieldMask { + paths: vec!["labels".to_string()], + }), + }; + let req = with_namespace!(req, self.namespace); + let info = ContentClient::new(self.inner.clone()) + .update(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .info + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to update info for content {}", + info.digest + )) + })?; + Ok(info) + }) + } + + pub fn get_image(&self, image_name: impl ToString) -> Result { self.rt.block_on(async { let name = image_name.to_string(); let req = GetImageRequest { name }; let req = with_namespace!(req, self.namespace); - let digest = ImagesClient::new(self.inner.clone()) + let image = ImagesClient::new(self.inner.clone()) .get(req) .await .map_err(|err| ShimError::Containerd(err.to_string()))? @@ -75,28 +235,57 @@ impl Client { .image .ok_or_else(|| { ShimError::Containerd(format!( - "failed to get image content sha for image {}", + "failed to get image for image {}", image_name.to_string() )) - })? - .target + })?; + Ok(image) + }) + } + + pub fn update_image(&self, image: Image) -> Result { + self.rt.block_on(async { + let req = UpdateImageRequest { + image: Some(image.clone()), + update_mask: Some(FieldMask { + paths: vec!["labels".to_string()], + }), + }; + let req = with_namespace!(req, self.namespace); + let image = ImagesClient::new(self.inner.clone()) + .update(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .image .ok_or_else(|| { - ShimError::Containerd(format!( - "failed to get image content sha for image {}", - image_name.to_string() - )) - })? - .digest; - Ok(digest) + ShimError::Containerd(format!("failed to update image {}", image.name)) + })?; + Ok(image) }) } - pub fn get_image(&self, container_name: impl ToString) -> Result { + pub fn extract_image_content_sha(&self, image: &Image) -> Result { + let digest = image + .target + .as_ref() + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image content sha for image {}", + image.name + )) + })? + .digest + .clone(); + Ok(digest) + } + + pub fn get_container(&self, container_name: impl ToString) -> Result { self.rt.block_on(async { let id = container_name.to_string(); let req = GetContainerRequest { id }; let req = with_namespace!(req, self.namespace); - let image = ContainersClient::new(self.inner.clone()) + let container = ContainersClient::new(self.inner.clone()) .get(req) .await .map_err(|err| ShimError::Containerd(err.to_string()))? @@ -107,23 +296,23 @@ impl Client { "failed to get image for container {}", container_name.to_string() )) - })? - .image; - Ok(image) + })?; + Ok(container) }) } // load module will query the containerd store to find an image that has an OS of type 'wasm' // If found it continues to parse the manifest and return the layers that contains the WASM modules // and possibly other configuration layers. - pub fn load_modules( + pub fn load_modules( &self, containerd_id: impl ToString, - supported_layer_types: &[&str], + engine: T, ) -> Result<(Vec, Platform)> { - let image_name = self.get_image(containerd_id.to_string())?; - let digest = self.get_image_content_sha(image_name)?; - let manifest = self.read_content(digest)?; + let container = self.get_container(containerd_id.to_string())?; + let mut image = self.get_image(container.image)?; + let digest = self.extract_image_content_sha(&image)?; + let manifest = self.read_content(digest.clone())?; let manifest = manifest.as_slice(); let manifest = ImageManifest::from_reader(manifest)?; @@ -137,23 +326,113 @@ impl Client { log::info!("manifest is not in WASM OCI image format"); return Ok((vec![], platform)); }; + log::info!("found manifest with WASM OCI image format."); + let label = format!("runwasi.io/precompiled/{}", T::name()); + match image.labels.get(&label) { + Some(precompile_digest) if T::can_precompile() => { + log::info!("found precompiled image"); + let precompiled = self.read_content(precompile_digest)?; + Ok(( + vec![WasmLayer { + config: image_config_descriptor.clone(), + layer: precompiled, + precompiled: true, + }], + platform, + )) + } + None if T::can_precompile() => { + log::info!("precompiling module"); + let layers = manifest + .layers() + .iter() + .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) + .map(|config| self.read_content(config.digest())) + .collect::>>()?; + + log::debug!("precompile complete and saving content"); + let precompiled = engine.precompile(layers.as_slice())?; + let precompile_digest = self.save_content(precompiled.clone())?; + + log::debug!("updating image with compiled content digest"); + image.labels.insert( + "runwasi.io/precompiled".to_string(), + precompile_digest.clone(), + ); + self.update_image(image)?; + + log::debug!("updating content with precompile digest to avoid garbage collection"); + let mut image_content = self.get_info(digest.clone())?; + image_content.labels.insert( + "containerd.io/gc.ref.content.precompile".to_string(), + precompile_digest.clone(), + ); + self.update_info(image_content)?; - let layers = manifest - .layers() - .iter() - .filter(|x| is_wasm_layer(x.media_type(), supported_layer_types)) - .map(|config| { - self.read_content(config.digest()).map(|module| WasmLayer { - config: config.clone(), - layer: module, - }) - }) - .collect::>>()?; - Ok((layers, platform)) + Ok(( + vec![WasmLayer { + config: image_config_descriptor.clone(), + layer: precompiled, + precompiled: true, + }], + platform, + )) + } + _ => { + log::info!("using module from OCI layers"); + let layers = manifest + .layers() + .iter() + .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) + .map(|config| { + self.read_content(config.digest()).map(|module| WasmLayer { + config: config.clone(), + layer: module, + precompiled: false, + }) + }) + .collect::>>()?; + Ok((layers, platform)) + } + } } } fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { supported_layer_types.contains(&media_type.to_string().as_str()) } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + + #[test] + fn test_save_content() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, "test-ns").unwrap(); + let data = b"hello world".to_vec(); + + let expected = digest(data.clone()); + let expected = format!("sha256:{}", expected); + + let returned = client.save_content(data).unwrap(); + assert_eq!(expected, returned); + + let data = client.read_content(returned).unwrap(); + assert_eq!(data, b"hello world"); + + // a second call should be successful since it already exists + let returned = client.save_content(data).unwrap(); + assert_eq!(expected, returned); + + client.delete_content(expected.clone()).unwrap(); + + client + .read_content(expected) + .expect_err("content should not exist"); + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index c7edfcf55..e3fa2cda3 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -15,6 +15,7 @@ use super::error::Result; pub struct WasmLayer { pub config: Descriptor, pub layer: Vec, + pub precompiled: bool, } fn parse_env(envs: &[String]) -> HashMap { diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index 8b228066f..b8142d1f0 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -45,7 +45,7 @@ impl SandboxInstance for Instance { // check if container is OCI image with wasm layers and attempt to read the module let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address(), &namespace)? - .load_modules(&id, E::supported_layers_types()) + .load_modules(&id, engine.clone()) .unwrap_or_else(|e| { log::warn!("Error obtaining wasm layers for container {id}. Will attempt to use files inside container image. Error: {e}"); (vec![], Platform::default()) diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index b1b10dda8..16eac71b5 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -9,6 +9,8 @@ use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; use wasmtime::{Module, Store}; use wasmtime_wasi::preview2::{self as wasi_preview2}; use wasmtime_wasi::{self as wasi_preview1, Dir}; +use wasmtime::{Linker, Module, Precompiled, Store}; +use wasmtime_wasi::{Dir, WasiCtxBuilder}; pub type WasmtimeInstance = Instance; @@ -61,6 +63,8 @@ impl Engine for WasmtimeEngine { } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { + let _config = wasmtime::Config::new(); + log::info!("setting up wasi"); let envs: Vec<_> = std::env::vars().collect(); let Entrypoint { @@ -96,6 +100,32 @@ impl Engine for WasmtimeEngine { Ok(status) } + + fn precompile(&self, layers: &[Vec]) -> Result> { + match layers { + [layer] => self.engine.precompile_module(layer), + _ => bail!("only a single module is supported when when precompiling"), + } + } + + fn can_precompile() -> bool { + true + } +} + +fn load_precompiled(engine: &wasmtime::Engine, bytes: &Vec) -> Result { + match engine.detect_precompiled(bytes) { + Some(Precompiled::Module) => { + log::info!("using precompiled module"); + unsafe { Module::deserialize(engine, bytes) } + } + Some(Precompiled::Component) => { + bail!("components not supported") + } + None => { + bail!("invalid precompiled module") + } + } } impl WasmtimeEngine { From 1476670d4b7ea2bc9b69205fd14472b51fcc5dba Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 12 Dec 2023 21:13:01 +0000 Subject: [PATCH 02/19] Add support for leases Signed-off-by: James Sturtevant --- .../src/container/engine.rs | 7 +- .../containerd-shim-wasm/src/container/mod.rs | 2 +- .../src/container/tests.rs | 8 +- .../src/sandbox/containerd.rs | 182 +++++++++++++++--- .../src/sys/unix/container/executor.rs | 4 +- .../src/sys/unix/container/instance.rs | 4 +- .../containerd-shim-wasmedge/src/instance.rs | 12 +- crates/containerd-shim-wasmer/src/instance.rs | 11 +- .../containerd-shim-wasmtime/src/instance.rs | 16 +- 9 files changed, 208 insertions(+), 38 deletions(-) diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 2b9119364..4a521b7c6 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -7,9 +7,14 @@ use super::Source; use crate::container::{PathResolve, RuntimeContext}; use crate::sandbox::Stdio; +pub struct RuntimeInfo { + pub name: &'static str, + pub version: &'static str, +} + pub trait Engine: Clone + Send + Sync + 'static { /// The name to use for this engine - fn name() -> &'static str; + fn info() -> &'static RuntimeInfo; /// Run a WebAssembly container fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result; diff --git a/crates/containerd-shim-wasm/src/container/mod.rs b/crates/containerd-shim-wasm/src/container/mod.rs index 64e58870b..a58c4d5a8 100644 --- a/crates/containerd-shim-wasm/src/container/mod.rs +++ b/crates/containerd-shim-wasm/src/container/mod.rs @@ -17,7 +17,7 @@ mod wasm; pub(crate) use context::WasiContext; pub use context::{Entrypoint, RuntimeContext, Source}; -pub use engine::Engine; +pub use engine::{Engine, RuntimeInfo}; pub use instance::Instance; pub use path::PathResolve; pub use wasm::WasmBinaryType; diff --git a/crates/containerd-shim-wasm/src/container/tests.rs b/crates/containerd-shim-wasm/src/container/tests.rs index bbf83869b..84080d69e 100644 --- a/crates/containerd-shim-wasm/src/container/tests.rs +++ b/crates/containerd-shim-wasm/src/container/tests.rs @@ -1,5 +1,6 @@ use anyhow::bail; +use super::engine::RuntimeInfo; use crate::container::{Engine, RuntimeContext, Stdio}; use crate::sys::container::instance::Instance; use crate::testing::WasiTest; @@ -8,8 +9,11 @@ use crate::testing::WasiTest; struct EngineFailingValidation; impl Engine for EngineFailingValidation { - fn name() -> &'static str { - "wasi_instance" + fn info() -> &'static RuntimeInfo { + &RuntimeInfo { + name: "wasi_instance", + version: "0.0.0", + } } fn can_handle(&self, _ctx: &impl RuntimeContext) -> anyhow::Result<()> { bail!("can't handle"); diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 190b23490..9166fe0f4 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -1,12 +1,12 @@ #![cfg(unix)] use std::collections::HashMap; -use std::path::Path; use containerd_client; use containerd_client::services::v1::containers_client::ContainersClient; use containerd_client::services::v1::content_client::ContentClient; use containerd_client::services::v1::images_client::ImagesClient; +use containerd_client::services::v1::leases_client::LeasesClient; use containerd_client::services::v1::{ Container, DeleteContentRequest, GetContainerRequest, GetImageRequest, Image, Info, InfoRequest, ReadContentRequest, UpdateImageRequest, UpdateRequest, WriteAction, @@ -23,7 +23,7 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Code, Request}; -use crate::container::Engine; +use crate::container::{Engine, RuntimeInfo}; use crate::sandbox::error::{Error as ShimError, Result}; use crate::sandbox::oci::{self, WasmLayer}; @@ -31,12 +31,68 @@ pub(crate) struct Client { inner: Channel, rt: Runtime, namespace: String, + address: String, +} + +// Adds lease info to grpc header +// https://github.com/containerd/containerd/blob/8459273f806e068e1a6bacfaf1355bbbad738d5e/docs/garbage-collection.md#using-grpc +#[macro_export] +macro_rules! with_lease { + ($req : ident, $ns: expr, $lease_id: expr) => {{ + let mut req = Request::new($req); + let md = req.metadata_mut(); + // https://github.com/containerd/containerd/blob/main/namespaces/grpc.go#L27 + md.insert("containerd-namespace", $ns.parse().unwrap()); + md.insert("containerd-lease", $lease_id.parse().unwrap()); + req + }}; +} + +struct LeaseGuard { + lease_id: String, + namespace: String, + address: String, +} + +pub(crate) struct WriteContent { + _lease: LeaseGuard, + pub digest: String, +} + +// Provides a best effort for dropping a lease of the content. If the lease cannot be dropped, it will log a warning +impl Drop for LeaseGuard { + fn drop(&mut self) { + let id = self.lease_id.clone(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let client = rt.block_on(containerd_client::connect(self.address.clone())); + + if client.is_err() { + log::warn!("failed to connect to containerd. lease may not be deleted"); + return; + } + + let mut client = LeasesClient::new(client.unwrap()); + + rt.block_on(async { + let req = containerd_client::services::v1::DeleteRequest { id, sync: false }; + let req = with_namespace!(req, self.namespace); + let result = client.delete(req).await; + + if result.is_err() { + log::warn!("failed to remove lease."); + } + }); + } } // sync wrapper implementation from https://tokio.rs/tokio/topics/bridging impl Client { // wrapper around connection that will establish a connection and create a client - pub fn connect(address: impl AsRef, namespace: impl ToString) -> Result { + pub fn connect(address: &str, namespace: impl ToString) -> Result { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; @@ -49,6 +105,7 @@ impl Client { inner, rt, namespace: namespace.to_string(), + address: address.to_string(), }) } @@ -88,18 +145,57 @@ impl Client { }) } - pub fn save_content(&self, data: Vec) -> Result { + // wrapper around lease that will create a lease and return a guard that will delete the lease when dropped + fn lease(&self, r#ref: String) -> Result { self.rt.block_on(async { + let mut lease_labels = HashMap::new(); + let expire = chrono::Utc::now() + chrono::Duration::hours(24); + lease_labels.insert("containerd.io/gc.expire".to_string(), expire.to_rfc3339()); + let lease_request = containerd_client::services::v1::CreateRequest { + id: r#ref.clone(), + labels: lease_labels, + }; + + let mut leases_client = LeasesClient::new(self.inner.clone()); + + let lease = leases_client + .create(with_namespace!(lease_request, self.namespace)) + .await + .map_err(|e| ShimError::Containerd(e.to_string()))? + .into_inner() + .lease + .ok_or_else(|| { + ShimError::Containerd(format!("unable to create lease for {}", r#ref.clone())) + })?; + + Ok(LeaseGuard { + lease_id: lease.id.clone(), + address: self.address.clone(), + namespace: self.namespace.clone(), + }) + }) + } + + pub fn save_content( + &self, + data: Vec, + original_digest: String, + info: &RuntimeInfo, + ) -> Result { + let expected = digest(data.clone()); + let expected = format!("sha256:{}", expected); + let r#ref = format!("precompile-{}-{}-{}", info.name, info.version, expected); + let lease = self.lease(r#ref.clone())?; + + let digest = self.rt.block_on(async { // create a channel to feed the stream; only sending one message at a time so we can set this to one let (tx, rx) = mpsc::channel(1); let len = data.len() as i64; - let expected = digest(data.clone()); - let expected = format!("sha256:{}", expected); + let mut client = ContentClient::new(self.inner.clone()); - let r#ref = "test".to_string(); - // Send Stat action to containerd to let it know that we are going to write content + // Send write request with Stat action to containerd to let it know that we are going to write content // if the content is already there, it will return early with AlreadyExists let req = WriteContentRequest { r#ref: r#ref.clone(), @@ -112,7 +208,8 @@ impl Client { .await .map_err(|err| ShimError::Containerd(err.to_string()))?; let request_stream = ReceiverStream::new(rx); - let request_stream = with_namespace!(request_stream, self.namespace); + let request_stream = + with_lease!(request_stream, self.namespace, lease.lease_id.clone()); let mut response_stream = match client.write(request_stream).await { Ok(response_stream) => response_stream.into_inner(), Err(e) if e.code() == Code::AlreadyExists => { @@ -134,7 +231,10 @@ impl Client { // Write and commit at same time let mut labels = HashMap::new(); - labels.insert("runwasi.io/precompiled".to_string(), "".to_string()); + labels.insert( + "runwasi.io/precompiled".to_string(), + original_digest.clone(), + ); let commit_request = WriteContentRequest { action: WriteAction::Commit.into(), total: len, @@ -172,6 +272,11 @@ impl Client { ))); } Ok(response.digest) + })?; + + Ok(WriteContent { + _lease: lease, + digest: digest.clone(), }) } @@ -311,8 +416,8 @@ impl Client { ) -> Result<(Vec, Platform)> { let container = self.get_container(containerd_id.to_string())?; let mut image = self.get_image(container.image)?; - let digest = self.extract_image_content_sha(&image)?; - let manifest = self.read_content(digest.clone())?; + let image_digest = self.extract_image_content_sha(&image)?; + let manifest = self.read_content(image_digest.clone())?; let manifest = manifest.as_slice(); let manifest = ImageManifest::from_reader(manifest)?; @@ -328,7 +433,13 @@ impl Client { }; log::info!("found manifest with WASM OCI image format."); - let label = format!("runwasi.io/precompiled/{}", T::name()); + // This label is unique across runtimes and versions + // a precompiled component/module will not work across different runtimes or versions + let label = format!( + "runwasi.io/precompiled/{}/{}", + T::info().name, + T::info().version + ); match image.labels.get(&label) { Some(precompile_digest) if T::can_precompile() => { log::info!("found precompiled image"); @@ -353,20 +464,24 @@ impl Client { log::debug!("precompile complete and saving content"); let precompiled = engine.precompile(layers.as_slice())?; - let precompile_digest = self.save_content(precompiled.clone())?; + let precompiled_content = + self.save_content(precompiled.clone(), image_digest.clone(), T::info())?; log::debug!("updating image with compiled content digest"); image.labels.insert( "runwasi.io/precompiled".to_string(), - precompile_digest.clone(), + precompiled_content.digest.clone(), ); self.update_image(image)?; + // The original image is considered a root object, by adding a ref to the new compiled content + // We tell to containerd to not garbage collect the new content until this image is removed from the system + // this ensures that we keep the content around after the lease is dropped log::debug!("updating content with precompile digest to avoid garbage collection"); - let mut image_content = self.get_info(digest.clone())?; + let mut image_content = self.get_info(image_digest.clone())?; image_content.labels.insert( "containerd.io/gc.ref.content.precompile".to_string(), - precompile_digest.clone(), + precompiled_content.digest.clone(), ); self.update_info(image_content)?; @@ -419,15 +534,36 @@ mod tests { let expected = digest(data.clone()); let expected = format!("sha256:{}", expected); - let returned = client.save_content(data).unwrap(); - assert_eq!(expected, returned); - - let data = client.read_content(returned).unwrap(); + let returned = client + .save_content( + data, + "original".to_string(), + &RuntimeInfo { + name: "test", + version: "0.0.0", + }, + ) + .unwrap(); + assert_eq!(expected, returned.digest.clone()); + + let data = client.read_content(returned.digest.clone()).unwrap(); assert_eq!(data, b"hello world"); + // need to drop the lease to be able to create a second one + drop(returned); + // a second call should be successful since it already exists - let returned = client.save_content(data).unwrap(); - assert_eq!(expected, returned); + let returned = client + .save_content( + data, + "original".to_string(), + &RuntimeInfo { + name: "test", + version: "0.0.0", + }, + ) + .unwrap(); + assert_eq!(expected, returned.digest); client.delete_content(expected.clone()).unwrap(); diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs index 81e45933e..2c943eccb 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs @@ -36,7 +36,7 @@ impl LibcontainerExecutor for Executor { fn validate(&self, spec: &Spec) -> Result<(), ExecutorValidationError> { // We can handle linux container. We delegate wasm container to the engine. match self.inner(spec) { - InnerExecutor::CantHandle => Err(ExecutorValidationError::CantHandle(E::name())), + InnerExecutor::CantHandle => Err(ExecutorValidationError::CantHandle(E::info().name)), _ => Ok(()), } } @@ -45,7 +45,7 @@ impl LibcontainerExecutor for Executor { // If it looks like a linux container, run it as a linux container. // Otherwise, run it as a wasm container match self.inner(spec) { - InnerExecutor::CantHandle => Err(LibcontainerExecutorError::CantHandle(E::name())), + InnerExecutor::CantHandle => Err(LibcontainerExecutorError::CantHandle(E::info().name)), InnerExecutor::Linux => { log::info!("executing linux container"); self.stdio.take().redirect().unwrap(); diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index b8142d1f0..d297b6b46 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -39,12 +39,12 @@ impl SandboxInstance for Instance { let engine = cfg.get_engine(); let bundle = cfg.get_bundle().to_path_buf(); let namespace = cfg.get_namespace(); - let rootdir = Path::new(DEFAULT_CONTAINER_ROOT_DIR).join(E::name()); + let rootdir = Path::new(DEFAULT_CONTAINER_ROOT_DIR).join(E::info().name); let rootdir = determine_rootdir(&bundle, &namespace, rootdir)?; let stdio = Stdio::init_from_cfg(cfg)?; // check if container is OCI image with wasm layers and attempt to read the module - let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address(), &namespace)? + let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address().as_str(), &namespace)? .load_modules(&id, engine.clone()) .unwrap_or_else(|e| { log::warn!("Error obtaining wasm layers for container {id}. Will attempt to use files inside container image. Error: {e}"); diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index 137c35f66..638439d4b 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -1,5 +1,10 @@ use anyhow::{Context, Result}; use containerd_shim_wasm::container::{Engine, Entrypoint, Instance, RuntimeContext, Stdio}; +use anyhow::{bail, Context, Result}; +use containerd_shim_wasm::container::{ + Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, +}; +use log::debug; use wasmedge_sdk::config::{ConfigBuilder, HostRegistrationConfigOptions}; use wasmedge_sdk::plugin::PluginManager; use wasmedge_sdk::VmBuilder; @@ -25,8 +30,11 @@ impl Default for WasmEdgeEngine { } impl Engine for WasmEdgeEngine { - fn name() -> &'static str { - "wasmedge" + fn info() -> &'static RuntimeInfo { + &RuntimeInfo { + name: "wasmedge", + version: env!("CARGO_PKG_VERSION"), + } } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { diff --git a/crates/containerd-shim-wasmer/src/instance.rs b/crates/containerd-shim-wasmer/src/instance.rs index 34fdb0104..ea615a880 100644 --- a/crates/containerd-shim-wasmer/src/instance.rs +++ b/crates/containerd-shim-wasmer/src/instance.rs @@ -1,5 +1,9 @@ use anyhow::Result; use containerd_shim_wasm::container::{Engine, Entrypoint, Instance, RuntimeContext, Stdio}; +use anyhow::{bail, Context, Result}; +use containerd_shim_wasm::container::{ + Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, +}; use wasmer::{Module, Store}; use wasmer_wasix::virtual_fs::host_fs::FileSystem; use wasmer_wasix::{WasiEnv, WasiError}; @@ -12,8 +16,11 @@ pub struct WasmerEngine { } impl Engine for WasmerEngine { - fn name() -> &'static str { - "wasmer" + fn info() -> &'static RuntimeInfo { + &RuntimeInfo { + name: "wasmer", + version: env!("CARGO_PKG_VERSION"), + } } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 16eac71b5..dfaa81b73 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -2,8 +2,10 @@ use std::fs::File; use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, RuntimeContext, Stdio, WasmBinaryType, + Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, WasmBinaryType, }; +use std::sync::OnceLock; + use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; use wasmtime::{Module, Store}; @@ -57,9 +59,17 @@ impl wasmtime_wasi::preview2::WasiView for WasiCtx { } } +fn version_info() -> &'static RuntimeInfo { + static INFO: OnceLock = OnceLock::new(); + INFO.get_or_init(|| RuntimeInfo { + name: "wasmtime", + version: option_env!("WASMTIME_VERSION_INFO").unwrap_or(env!("CARGO_PKG_VERSION")), + }) +} + impl Engine for WasmtimeEngine { - fn name() -> &'static str { - "wasmtime" + fn info() -> &'static RuntimeInfo { + version_info() } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { From d930dfe9c44c1fff231b594121160acc68f3181f Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 17 Jan 2024 20:58:08 +0000 Subject: [PATCH 03/19] rebase and some feedback Signed-off-by: James Sturtevant --- .../src/sandbox/containerd.rs | 26 +++++++++---------- .../containerd-shim-wasmtime/src/instance.rs | 2 -- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 9166fe0f4..55bb894fd 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -1,6 +1,7 @@ #![cfg(unix)] use std::collections::HashMap; +use std::path::Path; use containerd_client; use containerd_client::services::v1::containers_client::ContainersClient; @@ -83,7 +84,7 @@ impl Drop for LeaseGuard { let result = client.delete(req).await; if result.is_err() { - log::warn!("failed to remove lease."); + log::warn!("failed to remove lease"); } }); } @@ -92,13 +93,13 @@ impl Drop for LeaseGuard { // sync wrapper implementation from https://tokio.rs/tokio/topics/bridging impl Client { // wrapper around connection that will establish a connection and create a client - pub fn connect(address: &str, namespace: impl ToString) -> Result { + pub fn connect(address: impl AsRef + ToString, namespace: impl ToString) -> Result { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; let inner = rt - .block_on(containerd_client::connect(address)) + .block_on(containerd_client::connect(address.as_ref())) .map_err(|err| ShimError::Containerd(err.to_string()))?; Ok(Client { @@ -146,13 +147,13 @@ impl Client { } // wrapper around lease that will create a lease and return a guard that will delete the lease when dropped - fn lease(&self, r#ref: String) -> Result { + fn lease(&self, reference: String) -> Result { self.rt.block_on(async { let mut lease_labels = HashMap::new(); let expire = chrono::Utc::now() + chrono::Duration::hours(24); lease_labels.insert("containerd.io/gc.expire".to_string(), expire.to_rfc3339()); let lease_request = containerd_client::services::v1::CreateRequest { - id: r#ref.clone(), + id: reference.clone(), labels: lease_labels, }; @@ -165,11 +166,11 @@ impl Client { .into_inner() .lease .ok_or_else(|| { - ShimError::Containerd(format!("unable to create lease for {}", r#ref.clone())) + ShimError::Containerd(format!("unable to create lease for {}", reference)) })?; Ok(LeaseGuard { - lease_id: lease.id.clone(), + lease_id: lease.id, address: self.address.clone(), namespace: self.namespace.clone(), }) @@ -182,10 +183,9 @@ impl Client { original_digest: String, info: &RuntimeInfo, ) -> Result { - let expected = digest(data.clone()); - let expected = format!("sha256:{}", expected); - let r#ref = format!("precompile-{}-{}-{}", info.name, info.version, expected); - let lease = self.lease(r#ref.clone())?; + let expected = format!("sha256:{}", digest(data.clone())); + let reference = format!("precompile-{}-{}-{}", info.name, info.version, expected); + let lease = self.lease(reference.clone())?; let digest = self.rt.block_on(async { // create a channel to feed the stream; only sending one message at a time so we can set this to one @@ -198,7 +198,7 @@ impl Client { // Send write request with Stat action to containerd to let it know that we are going to write content // if the content is already there, it will return early with AlreadyExists let req = WriteContentRequest { - r#ref: r#ref.clone(), + r#ref: reference.clone(), action: WriteAction::Stat.into(), total: len, expected: expected.clone(), @@ -225,7 +225,7 @@ impl Client { .ok_or_else(|| { ShimError::Containerd(format!( "no response received after write request for {}", - expected.clone() + expected )) })?; diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index dfaa81b73..a567a72a2 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -73,8 +73,6 @@ impl Engine for WasmtimeEngine { } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { - let _config = wasmtime::Config::new(); - log::info!("setting up wasi"); let envs: Vec<_> = std::env::vars().collect(); let Entrypoint { From 15a70febc3c4ff51e0140e3c8ab76d0c67bb7df4 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 17 Jan 2024 22:37:50 +0000 Subject: [PATCH 04/19] Refactor to support components Signed-off-by: James Sturtevant --- .../src/container/context.rs | 4 - .../src/container/engine.rs | 1 - .../src/sandbox/containerd.rs | 8 +- .../containerd-shim-wasm/src/sandbox/oci.rs | 1 - .../containerd-shim-wasmedge/src/instance.rs | 3 +- .../containerd-shim-wasmtime/src/instance.rs | 76 +++++++++++-------- 6 files changed, 49 insertions(+), 44 deletions(-) diff --git a/crates/containerd-shim-wasm/src/container/context.rs b/crates/containerd-shim-wasm/src/container/context.rs index f93a7e66d..5f4475a5b 100644 --- a/crates/containerd-shim-wasm/src/container/context.rs +++ b/crates/containerd-shim-wasm/src/container/context.rs @@ -45,7 +45,6 @@ pub enum Source<'a> { // and they will be included in this array, e.g., a `toml` file with the // runtime configuration. Oci(&'a [WasmLayer]), - Precompiled(&'a WasmLayer), } impl<'a> Source<'a> { @@ -101,8 +100,6 @@ impl RuntimeContext for WasiContext<'_> { let source = if self.wasm_layers.is_empty() { Source::File(PathBuf::from(path)) - } else if self.wasm_layers.len() == 1 && self.wasm_layers[0].precompiled { - Source::Precompiled(&self.wasm_layers[0]) } else { Source::Oci(self.wasm_layers) }; @@ -361,7 +358,6 @@ mod tests { wasm_layers: &[WasmLayer { layer: vec![], config: Descriptor::new(oci_spec::image::MediaType::Other("".to_string()), 10, ""), - precompiled: false, }], platform: &Platform::default(), }; diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 4a521b7c6..9bac45f12 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -31,7 +31,6 @@ pub trait Engine: Clone + Send + Sync + 'static { let path = match source { Source::File(path) => path, Source::Oci(_) => return Ok(()), - Source::Precompiled(_) => return Ok(()), }; path.resolve_in_path_or_cwd() diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 55bb894fd..27b62e605 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -93,7 +93,10 @@ impl Drop for LeaseGuard { // sync wrapper implementation from https://tokio.rs/tokio/topics/bridging impl Client { // wrapper around connection that will establish a connection and create a client - pub fn connect(address: impl AsRef + ToString, namespace: impl ToString) -> Result { + pub fn connect( + address: impl AsRef + ToString, + namespace: impl ToString, + ) -> Result { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; @@ -448,7 +451,6 @@ impl Client { vec![WasmLayer { config: image_config_descriptor.clone(), layer: precompiled, - precompiled: true, }], platform, )) @@ -489,7 +491,6 @@ impl Client { vec![WasmLayer { config: image_config_descriptor.clone(), layer: precompiled, - precompiled: true, }], platform, )) @@ -504,7 +505,6 @@ impl Client { self.read_content(config.digest()).map(|module| WasmLayer { config: config.clone(), layer: module, - precompiled: false, }) }) .collect::>>()?; diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index e3fa2cda3..c7edfcf55 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -15,7 +15,6 @@ use super::error::Result; pub struct WasmLayer { pub config: Descriptor, pub layer: Vec, - pub precompiled: bool, } fn parse_env(envs: &[String]) -> HashMap { diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index 638439d4b..610acd481 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -1,6 +1,5 @@ use anyhow::{Context, Result}; -use containerd_shim_wasm::container::{Engine, Entrypoint, Instance, RuntimeContext, Stdio}; -use anyhow::{bail, Context, Result}; +use anyhow::{bail}; use containerd_shim_wasm::container::{ Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, }; diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index a567a72a2..cc02aaa9b 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,18 +1,17 @@ +use std::borrow::Cow; use std::fs::File; +use std::sync::OnceLock; use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, WasmBinaryType, + Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, + WasmBinaryType, }; -use std::sync::OnceLock; - use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; -use wasmtime::{Module, Store}; -use wasmtime_wasi::preview2::{self as wasi_preview2}; +use wasmtime::{Module, Precompiled, Store}; +use wasmtime_wasi::preview2::{self as wasi_preview2, Table}; use wasmtime_wasi::{self as wasi_preview1, Dir}; -use wasmtime::{Linker, Module, Precompiled, Store}; -use wasmtime_wasi::{Dir, WasiCtxBuilder}; pub type WasmtimeInstance = Instance; @@ -89,11 +88,7 @@ impl Engine for WasmtimeEngine { let store = Store::new(&self.engine, wasi_ctx); let wasm_bytes = &source.as_bytes()?; - let status = match WasmBinaryType::from_bytes(wasm_bytes) { - Some(WasmBinaryType::Module) => self.execute_module(wasm_bytes, store, &func)?, - Some(WasmBinaryType::Component) => self.execute_component(wasm_bytes, store, func)?, - None => bail!("not a valid wasm binary format"), - }; + let status = self.execute(wasm_bytes, store, func)?; let status = status.map(|_| 0).or_else(|err| { match err.downcast_ref::() { @@ -121,21 +116,6 @@ impl Engine for WasmtimeEngine { } } -fn load_precompiled(engine: &wasmtime::Engine, bytes: &Vec) -> Result { - match engine.detect_precompiled(bytes) { - Some(Precompiled::Module) => { - log::info!("using precompiled module"); - unsafe { Module::deserialize(engine, bytes) } - } - Some(Precompiled::Component) => { - bail!("components not supported") - } - None => { - bail!("invalid precompiled module") - } - } -} - impl WasmtimeEngine { /// Execute a wasm module. /// @@ -143,12 +123,10 @@ impl WasmtimeEngine { /// to execute a wasm module that uses wasi_preview1. fn execute_module( &self, - wasm_binary: &[u8], + module: Module, mut store: Store, func: &String, ) -> Result, anyhow::Error> { - log::debug!("loading wasm module"); - let module = Module::from_binary(&self.engine, wasm_binary)?; let mut module_linker = wasmtime::Linker::new(&self.engine); wasi_preview1::add_to_linker(&mut module_linker, |s: &mut WasiCtx| &mut s.wasi_preview1)?; @@ -172,12 +150,12 @@ impl WasmtimeEngine { /// to execute a wasm component that uses wasi_preview2. fn execute_component( &self, - wasm_binary: &[u8], + component: Component, mut store: Store, func: String, ) -> Result, anyhow::Error> { log::debug!("loading wasm component"); - let component = Component::from_binary(&self.engine, wasm_binary)?; + let mut linker = wasmtime_component::Linker::new(&self.engine); wasi_preview2::command::sync::add_to_linker(&mut linker)?; @@ -209,6 +187,40 @@ impl WasmtimeEngine { Ok(status) } } + + fn execute( + &self, + wasm_binary: &Cow<'_, [u8]>, + store: Store, + func: String, + ) -> Result, anyhow::Error> { + match WasmBinaryType::from_bytes(wasm_binary) { + Some(WasmBinaryType::Module) => { + log::debug!("loading wasm module"); + let module = Module::from_binary(&self.engine, wasm_binary)?; + self.execute_module(module, store, &func) + } + Some(WasmBinaryType::Component) => { + let component = Component::from_binary(&self.engine, wasm_binary)?; + self.execute_component(component, store, func) + } + None => match &self.engine.detect_precompiled(wasm_binary) { + Some(Precompiled::Module) => { + log::info!("using precompiled module"); + let module = unsafe { Module::deserialize(&self.engine, wasm_binary) }?; + self.execute_module(module, store, &func) + } + Some(Precompiled::Component) => { + log::info!("using precompiled module"); + let component = unsafe { Component::deserialize(&self.engine, wasm_binary) }?; + self.execute_component(component, store, func) + } + None => { + bail!("invalid precompiled module") + } + }, + } + } } /// Prepare both wasi_preview1 and wasi_preview2 contexts. From 1d56e5dd9ce1ba6babc44d5c35015851983ecd0c Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 18 Jan 2024 00:50:19 +0000 Subject: [PATCH 05/19] Fix issue with using different labels Signed-off-by: James Sturtevant --- .../src/sandbox/containerd.rs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 27b62e605..bd84fef2c 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -232,10 +232,11 @@ impl Client { )) })?; + let label = precompile_label(info); // Write and commit at same time let mut labels = HashMap::new(); labels.insert( - "runwasi.io/precompiled".to_string(), + label, original_digest.clone(), ); let commit_request = WriteContentRequest { @@ -438,11 +439,7 @@ impl Client { log::info!("found manifest with WASM OCI image format."); // This label is unique across runtimes and versions // a precompiled component/module will not work across different runtimes or versions - let label = format!( - "runwasi.io/precompiled/{}/{}", - T::info().name, - T::info().version - ); + let label = precompile_label(T::info()); match image.labels.get(&label) { Some(precompile_digest) if T::can_precompile() => { log::info!("found precompiled image"); @@ -471,7 +468,7 @@ impl Client { log::debug!("updating image with compiled content digest"); image.labels.insert( - "runwasi.io/precompiled".to_string(), + label, precompiled_content.digest.clone(), ); self.update_image(image)?; @@ -514,10 +511,20 @@ impl Client { } } +fn precompile_label(info: &RuntimeInfo) -> String { + let label = format!( + "runwasi.io/precompiled/{}/{}", + info.name, + info.version + ); + label +} + fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { supported_layer_types.contains(&media_type.to_string().as_str()) } + #[cfg(test)] mod tests { use std::path::PathBuf; From 9270210b2ffd046edd93a5012c61b05d2c8b7ea0 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 18 Jan 2024 01:07:56 +0000 Subject: [PATCH 06/19] Addressing more feedback Signed-off-by: James Sturtevant --- .../src/container/engine.rs | 2 + .../src/sandbox/containerd.rs | 144 ++++++++++-------- 2 files changed, 85 insertions(+), 61 deletions(-) diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 9bac45f12..b8dde17aa 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -7,6 +7,8 @@ use super::Source; use crate::container::{PathResolve, RuntimeContext}; use crate::sandbox::Stdio; +/// RuntimeInfo contains the name and version of the runtime that is running +#[derive(Clone, Debug, Default)] pub struct RuntimeInfo { pub name: &'static str, pub version: &'static str, diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index bd84fef2c..003c467f7 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -24,9 +24,13 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Code, Request}; +use self::lease::LeaseGuard; use crate::container::{Engine, RuntimeInfo}; use crate::sandbox::error::{Error as ShimError, Result}; use crate::sandbox::oci::{self, WasmLayer}; +use crate::with_lease; + +static PRECOMPILE_PREFIX: &str = "runwasi.io/precompiled"; pub(crate) struct Client { inner: Channel, @@ -35,61 +39,77 @@ pub(crate) struct Client { address: String, } -// Adds lease info to grpc header -// https://github.com/containerd/containerd/blob/8459273f806e068e1a6bacfaf1355bbbad738d5e/docs/garbage-collection.md#using-grpc -#[macro_export] -macro_rules! with_lease { - ($req : ident, $ns: expr, $lease_id: expr) => {{ - let mut req = Request::new($req); - let md = req.metadata_mut(); - // https://github.com/containerd/containerd/blob/main/namespaces/grpc.go#L27 - md.insert("containerd-namespace", $ns.parse().unwrap()); - md.insert("containerd-lease", $lease_id.parse().unwrap()); - req - }}; -} - -struct LeaseGuard { - lease_id: String, - namespace: String, - address: String, -} +mod lease { + use containerd_client::services::v1::leases_client::LeasesClient; + use containerd_client::with_namespace; -pub(crate) struct WriteContent { - _lease: LeaseGuard, - pub digest: String, -} + use super::*; -// Provides a best effort for dropping a lease of the content. If the lease cannot be dropped, it will log a warning -impl Drop for LeaseGuard { - fn drop(&mut self) { - let id = self.lease_id.clone(); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); + // Adds lease info to grpc header + // https://github.com/containerd/containerd/blob/8459273f806e068e1a6bacfaf1355bbbad738d5e/docs/garbage-collection.md#using-grpc + #[macro_export] + macro_rules! with_lease { + ($req : ident, $ns: expr, $lease_id: expr) => {{ + let mut req = Request::new($req); + let md = req.metadata_mut(); + // https://github.com/containerd/containerd/blob/main/namespaces/grpc.go#L27 + md.insert("containerd-namespace", $ns.parse().unwrap()); + md.insert("containerd-lease", $lease_id.parse().unwrap()); + req + }}; + } - let client = rt.block_on(containerd_client::connect(self.address.clone())); + #[derive(Debug)] + pub(crate) struct LeaseGuard { + pub(crate) lease_id: String, + pub(crate) namespace: String, + pub(crate) address: String, + } - if client.is_err() { - log::warn!("failed to connect to containerd. lease may not be deleted"); - return; - } + // Provides a best effort for dropping a lease of the content. If the lease cannot be dropped, it will log a warning + impl Drop for LeaseGuard { + fn drop(&mut self) { + let id = self.lease_id.clone(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let client = rt.block_on(containerd_client::connect(self.address.clone())); + + let channel = match client { + Ok(channel) => channel, + Err(e) => { + log::error!( + "failed to connect to containerd: {}. lease may not be deleted", + e + ); + return; + } + }; - let mut client = LeasesClient::new(client.unwrap()); + let mut client = LeasesClient::new(channel); - rt.block_on(async { - let req = containerd_client::services::v1::DeleteRequest { id, sync: false }; - let req = with_namespace!(req, self.namespace); - let result = client.delete(req).await; + rt.block_on(async { + let req = containerd_client::services::v1::DeleteRequest { id, sync: false }; + let req = with_namespace!(req, self.namespace); + let result = client.delete(req).await; - if result.is_err() { - log::warn!("failed to remove lease"); - } - }); + match result { + Ok(_) => log::debug!("removed lease"), + Err(e) => log::error!("failed to remove lease: {}", e), + } + }); + } } } +#[derive(Debug)] +pub(crate) struct WriteContent { + _lease: LeaseGuard, + pub digest: String, +} + // sync wrapper implementation from https://tokio.rs/tokio/topics/bridging impl Client { // wrapper around connection that will establish a connection and create a client @@ -150,7 +170,7 @@ impl Client { } // wrapper around lease that will create a lease and return a guard that will delete the lease when dropped - fn lease(&self, reference: String) -> Result { + fn lease(&self, reference: String) -> Result { self.rt.block_on(async { let mut lease_labels = HashMap::new(); let expire = chrono::Utc::now() + chrono::Duration::hours(24); @@ -235,10 +255,7 @@ impl Client { let label = precompile_label(info); // Write and commit at same time let mut labels = HashMap::new(); - labels.insert( - label, - original_digest.clone(), - ); + labels.insert(label, original_digest.clone()); let commit_request = WriteContentRequest { action: WriteAction::Commit.into(), total: len, @@ -467,10 +484,9 @@ impl Client { self.save_content(precompiled.clone(), image_digest.clone(), T::info())?; log::debug!("updating image with compiled content digest"); - image.labels.insert( - label, - precompiled_content.digest.clone(), - ); + image + .labels + .insert(label, precompiled_content.digest.clone()); self.update_image(image)?; // The original image is considered a root object, by adding a ref to the new compiled content @@ -512,19 +528,13 @@ impl Client { } fn precompile_label(info: &RuntimeInfo) -> String { - let label = format!( - "runwasi.io/precompiled/{}/{}", - info.name, - info.version - ); - label + format!("{}/{}/{}", PRECOMPILE_PREFIX, info.name, info.version) } fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { supported_layer_types.contains(&media_type.to_string().as_str()) } - #[cfg(test)] mod tests { use std::path::PathBuf; @@ -556,7 +566,19 @@ mod tests { let data = client.read_content(returned.digest.clone()).unwrap(); assert_eq!(data, b"hello world"); + client + .save_content( + data.clone(), + "original".to_string(), + &RuntimeInfo { + name: "test", + version: "0.0.0", + }, + ) + .expect_err("Should not be able to save when lease is open"); + // need to drop the lease to be able to create a second one + // a second call should be successful since it already exists drop(returned); // a second call should be successful since it already exists From 308db362f95a6b9708e46320544cc33e896a21b9 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 18 Jan 2024 21:58:42 +0000 Subject: [PATCH 07/19] Add integration test to ensure lables are set Signed-off-by: James Sturtevant --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index 11aeec473..72a076db5 100644 --- a/Makefile +++ b/Makefile @@ -241,6 +241,13 @@ test/k8s/deploy-workload-oci-%: test/k8s/clean test/k8s/cluster-oci-% # verify that we are still running after some time sleep 5s kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=5s + @if [ "$*" = "wasmtime" ]; then \ + set -e; \ + echo "checking for pre-compiled label and ensuring can scale"; \ + docker exec $(KIND_CLUSTER_NAME)-control-plane ctr -n k8s.io i ls | grep "runwasi.io/precompiled"; \ + kubectl --context=kind-$(KIND_CLUSTER_NAME) scale deployment wasi-demo --replicas=4; \ + kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=5s; \ + fi .PHONY: test/k8s-% test/k8s-%: test/k8s/deploy-workload-% @@ -292,6 +299,13 @@ test/k3s-oci-%: dist/img-oci.tar bin/k3s dist-% sleep 5s sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=5s sudo bin/k3s kubectl get pods -o wide + @if [ "$*" = "wasmtime" ]; then \ + set -e; \ + echo "checking for pre-compiled label and ensuring can scale"; \ + sudo bin/k3s ctr -n k8s.io i ls | grep "runwasi.io/precompiled"; \ + sudo bin/k3s kubectl scale deployment wasi-demo --replicas=4; \ + sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=5s; \ + fi sudo bin/k3s kubectl delete -f test/k8s/deploy.oci.yaml sudo bin/k3s kubectl wait deployment wasi-demo --for delete --timeout=60s @@ -302,6 +316,7 @@ test/k3s/clean: bin/k3s/clean; clean: -rm -rf dist -rm -rf bin + -rm -rf test/k8s/_out -$(MAKE) test-image/clean -$(MAKE) test/k8s/clean -$(MAKE) test/k3s/clean From dba92e0dca3a27df2a5b72cc5ba1af913fb2c5a5 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 19 Jan 2024 00:21:39 +0000 Subject: [PATCH 08/19] Refactor and add documentation Signed-off-by: James Sturtevant --- .../src/sandbox/containerd.rs | 112 +++++++++--------- docs/oci-descision-flow.md | 47 ++++++++ 2 files changed, 101 insertions(+), 58 deletions(-) create mode 100644 docs/oci-descision-flow.md diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 003c467f7..8fd12a6f6 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -454,76 +454,72 @@ impl Client { }; log::info!("found manifest with WASM OCI image format."); - // This label is unique across runtimes and versions + // This label is unique across runtimes and version of the shim running // a precompiled component/module will not work across different runtimes or versions let label = precompile_label(T::info()); match image.labels.get(&label) { Some(precompile_digest) if T::can_precompile() => { - log::info!("found precompiled image"); + log::info!("found precompiled module in cache"); let precompiled = self.read_content(precompile_digest)?; - Ok(( + return Ok(( vec![WasmLayer { config: image_config_descriptor.clone(), layer: precompiled, }], platform, - )) - } - None if T::can_precompile() => { - log::info!("precompiling module"); - let layers = manifest - .layers() - .iter() - .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) - .map(|config| self.read_content(config.digest())) - .collect::>>()?; - - log::debug!("precompile complete and saving content"); - let precompiled = engine.precompile(layers.as_slice())?; - let precompiled_content = - self.save_content(precompiled.clone(), image_digest.clone(), T::info())?; - - log::debug!("updating image with compiled content digest"); - image - .labels - .insert(label, precompiled_content.digest.clone()); - self.update_image(image)?; - - // The original image is considered a root object, by adding a ref to the new compiled content - // We tell to containerd to not garbage collect the new content until this image is removed from the system - // this ensures that we keep the content around after the lease is dropped - log::debug!("updating content with precompile digest to avoid garbage collection"); - let mut image_content = self.get_info(image_digest.clone())?; - image_content.labels.insert( - "containerd.io/gc.ref.content.precompile".to_string(), - precompiled_content.digest.clone(), - ); - self.update_info(image_content)?; - - Ok(( - vec![WasmLayer { - config: image_config_descriptor.clone(), - layer: precompiled, - }], - platform, - )) - } - _ => { - log::info!("using module from OCI layers"); - let layers = manifest - .layers() - .iter() - .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) - .map(|config| { - self.read_content(config.digest()).map(|module| WasmLayer { - config: config.clone(), - layer: module, - }) - }) - .collect::>>()?; - Ok((layers, platform)) + )); } + _ => {} + } + + let layers = manifest + .layers() + .iter() + .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) + .map(|config| self.read_content(config.digest())) + .collect::>>()?; + + if T::can_precompile() { + log::info!("precompiling module"); + let precompiled = engine.precompile(layers.as_slice())?; + let precompiled_content = + self.save_content(precompiled.clone(), image_digest.clone(), T::info())?; + + log::debug!("updating image with compiled content digest"); + image + .labels + .insert(label, precompiled_content.digest.clone()); + self.update_image(image)?; + + // The original image is considered a root object, by adding a ref to the new compiled content + // We tell to containerd to not garbage collect the new content until this image is removed from the system + // this ensures that we keep the content around after the lease is dropped + log::debug!("updating content with precompile digest to avoid garbage collection"); + let mut image_content = self.get_info(image_digest.clone())?; + image_content.labels.insert( + "containerd.io/gc.ref.content.precompile".to_string(), + precompiled_content.digest.clone(), + ); + self.update_info(image_content)?; + + return Ok(( + vec![WasmLayer { + config: image_config_descriptor.clone(), + layer: precompiled, + }], + platform, + )); } + + log::info!("using module from OCI layers"); + let layers = layers + .into_iter() + .map(|module| WasmLayer { + config: image_config_descriptor.clone(), + layer: module, + }) + .collect::>(); + Ok((layers, platform)) } } diff --git a/docs/oci-descision-flow.md b/docs/oci-descision-flow.md new file mode 100644 index 000000000..ac4525921 --- /dev/null +++ b/docs/oci-descision-flow.md @@ -0,0 +1,47 @@ +# OCI pre-compilation + +The OCI images layers are loaded from containerd. If the runtime supports pre-compilation the images will be precompiled and cached using the containerd content store. + +```mermaid +graph TD + start[Task new] + imgconfig[Load image config from containerd] + iswasm{Arch==wasm?} + alreadycompiled{Does image label for shim runtime version exist? runwasi.io/precompiled/runtime/version} + startcontainer[Create Container] + precompiledenabled{Is precompiling enabled in shim?} + precompiledenabled2{Is precompiling enabled in shim?} + fetchcache[Fetch cached precompiled layer from containerd content store] + precompile[Precompile using wasm runtime] + loadoci[Load OCI layers from containerd] + storecache[Store precompiled layer in containerd content store] + + start --> imgconfig --> iswasm + iswasm -- yes --> precompiledenabled + iswasm -- no. wasm will be loaded from file inside image --> startcontainer + + precompiledenabled -- yes --> alreadycompiled + precompiledenabled -- no --> loadoci --> precompiledenabled2 + + alreadycompiled -- yes --> fetchcache --> startcontainer + alreadycompiled -- no --> loadoci + + precompiledenabled2 -- yes --> precompile --> storecache --> startcontainer + precompiledenabled2 -- no --> startcontainer +``` + +Once a wasm module or component is pre-compiled it will remain in the containerd content store until the original image is removed from containerd. There is a small disk overhead associated with this but it reduces the complexity of managing stored versions during upgrades. + +To view the images in containerd that have associated pre-compilations: + +```bash +sudo ctr i ls | grep "runwasi.io" +ghcr.io/containerd/runwasi/wasi-demo-oci:latest application/vnd.oci.image.manifest.v1+json + sha256:60fccd77070dfeb682a1ebc742e9d677fc452b30a6b99188b081c968992394ce 2.4 MiB wasi/wasm +runwasi.io/precompiled/wasmtime/0.3.1=sha256:b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f1398706782e225fd0a98e + +# query for the sha in the label +sudo ctr content ls | grep "b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f139870" +sha256:60fccd77070dfeb682a1ebc742e9d677fc452b30a6b99188b081c968992394ce 561B 2 months containerd.io/gc.ref.content.0=sha256:a3c18cd551d54d3cfbf67acc9e8f7ef5761e76827fe7c1ae163fca0193be88b3,containerd.io/gc.ref.content.config=sha256:85b7f2b562fe8665ec9d9e6d47ab0b24e2315627f5f558d298475c4038d71e8b,containerd.io/gc.ref.content.precompile=sha256:b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f1398706782e225fd0a98e +sha256:b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f1398706782e225fd0a98e 626.4kB 3 days runwasi.io/precompiled=sha256:60fccd77070dfeb682a1ebc742e9d677fc452b30a6b99188b081c968992394ce +``` \ No newline at end of file From e5d73b166b1b3e88e47c56a50f93e63246ca63d8 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 24 Jan 2024 17:04:11 +0000 Subject: [PATCH 09/19] exit early if layers WASM layers not found Signed-off-by: James Sturtevant --- .../src/container/engine.rs | 5 +- .../{containerd.rs => containerd/client.rs} | 92 ++++--------------- .../src/sandbox/containerd/lease.rs | 61 ++++++++++++ .../src/sandbox/containerd/mod.rs | 4 + .../containerd-shim-wasmedge/src/instance.rs | 4 +- crates/containerd-shim-wasmer/src/instance.rs | 4 +- .../containerd-shim-wasmtime/src/instance.rs | 6 +- 7 files changed, 89 insertions(+), 87 deletions(-) rename crates/containerd-shim-wasm/src/sandbox/{containerd.rs => containerd/client.rs} (86%) create mode 100644 crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs create mode 100644 crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index b8dde17aa..054f88664 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -60,7 +60,10 @@ pub trait Engine: Clone + Send + Sync + 'static { &["application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"] } - /// Precomiple a module + /// Precomiples a module that in in the WASM OCI layer format + /// This is used to precompile a module before it is run and will be called if can_precompile returns true. + /// It is called only the first time a module is run and the resulting bytes will be cached in the containerd content store. + /// The cached, precompiled module will be reloaded on subsequent runs. fn precompile(&self, _layers: &[Vec]) -> Result> { bail!("precompilation not supported for this runtime") } diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs similarity index 86% rename from crates/containerd-shim-wasm/src/sandbox/containerd.rs rename to crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index 8fd12a6f6..38e4cfeca 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -24,7 +24,7 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Code, Request}; -use self::lease::LeaseGuard; +use super::lease::LeaseGuard; use crate::container::{Engine, RuntimeInfo}; use crate::sandbox::error::{Error as ShimError, Result}; use crate::sandbox::oci::{self, WasmLayer}; @@ -32,78 +32,13 @@ use crate::with_lease; static PRECOMPILE_PREFIX: &str = "runwasi.io/precompiled"; -pub(crate) struct Client { +pub struct Client { inner: Channel, rt: Runtime, namespace: String, address: String, } -mod lease { - use containerd_client::services::v1::leases_client::LeasesClient; - use containerd_client::with_namespace; - - use super::*; - - // Adds lease info to grpc header - // https://github.com/containerd/containerd/blob/8459273f806e068e1a6bacfaf1355bbbad738d5e/docs/garbage-collection.md#using-grpc - #[macro_export] - macro_rules! with_lease { - ($req : ident, $ns: expr, $lease_id: expr) => {{ - let mut req = Request::new($req); - let md = req.metadata_mut(); - // https://github.com/containerd/containerd/blob/main/namespaces/grpc.go#L27 - md.insert("containerd-namespace", $ns.parse().unwrap()); - md.insert("containerd-lease", $lease_id.parse().unwrap()); - req - }}; - } - - #[derive(Debug)] - pub(crate) struct LeaseGuard { - pub(crate) lease_id: String, - pub(crate) namespace: String, - pub(crate) address: String, - } - - // Provides a best effort for dropping a lease of the content. If the lease cannot be dropped, it will log a warning - impl Drop for LeaseGuard { - fn drop(&mut self) { - let id = self.lease_id.clone(); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - let client = rt.block_on(containerd_client::connect(self.address.clone())); - - let channel = match client { - Ok(channel) => channel, - Err(e) => { - log::error!( - "failed to connect to containerd: {}. lease may not be deleted", - e - ); - return; - } - }; - - let mut client = LeasesClient::new(channel); - - rt.block_on(async { - let req = containerd_client::services::v1::DeleteRequest { id, sync: false }; - let req = with_namespace!(req, self.namespace); - let result = client.delete(req).await; - - match result { - Ok(_) => log::debug!("removed lease"), - Err(e) => log::error!("failed to remove lease: {}", e), - } - }); - } - } -} - #[derive(Debug)] pub(crate) struct WriteContent { _lease: LeaseGuard, @@ -134,7 +69,7 @@ impl Client { } // wrapper around read that will read the entire content file - pub fn read_content(&self, digest: impl ToString) -> Result> { + fn read_content(&self, digest: impl ToString) -> Result> { self.rt.block_on(async { let req = ReadContentRequest { digest: digest.to_string(), @@ -170,7 +105,7 @@ impl Client { } // wrapper around lease that will create a lease and return a guard that will delete the lease when dropped - fn lease(&self, reference: String) -> Result { + fn lease(&self, reference: String) -> Result { self.rt.block_on(async { let mut lease_labels = HashMap::new(); let expire = chrono::Utc::now() + chrono::Duration::hours(24); @@ -200,7 +135,7 @@ impl Client { }) } - pub fn save_content( + fn save_content( &self, data: Vec, original_digest: String, @@ -301,7 +236,7 @@ impl Client { }) } - pub fn get_info(&self, content_digest: String) -> Result { + fn get_info(&self, content_digest: String) -> Result { self.rt.block_on(async { let req = InfoRequest { digest: content_digest.clone(), @@ -323,7 +258,7 @@ impl Client { }) } - pub fn update_info(&self, info: Info) -> Result { + fn update_info(&self, info: Info) -> Result { self.rt.block_on(async { let req = UpdateRequest { info: Some(info.clone()), @@ -348,7 +283,7 @@ impl Client { }) } - pub fn get_image(&self, image_name: impl ToString) -> Result { + fn get_image(&self, image_name: impl ToString) -> Result { self.rt.block_on(async { let name = image_name.to_string(); let req = GetImageRequest { name }; @@ -369,7 +304,7 @@ impl Client { }) } - pub fn update_image(&self, image: Image) -> Result { + fn update_image(&self, image: Image) -> Result { self.rt.block_on(async { let req = UpdateImageRequest { image: Some(image.clone()), @@ -391,7 +326,7 @@ impl Client { }) } - pub fn extract_image_content_sha(&self, image: &Image) -> Result { + fn extract_image_content_sha(&self, image: &Image) -> Result { let digest = image .target .as_ref() @@ -406,7 +341,7 @@ impl Client { Ok(digest) } - pub fn get_container(&self, container_name: impl ToString) -> Result { + fn get_container(&self, container_name: impl ToString) -> Result { self.rt.block_on(async { let id = container_name.to_string(); let req = GetContainerRequest { id }; @@ -479,6 +414,11 @@ impl Client { .map(|config| self.read_content(config.digest())) .collect::>>()?; + if layers.is_empty() { + log::info!("no WASM modules found in OCI layers"); + return Ok((vec![], platform)); + } + if T::can_precompile() { log::info!("precompiling module"); let precompiled = engine.precompile(layers.as_slice())?; diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs new file mode 100644 index 000000000..83487c4cc --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs @@ -0,0 +1,61 @@ +use containerd_client::services::v1::leases_client::LeasesClient; +use containerd_client::{tonic, with_namespace}; +use tonic::Request; + +// Adds lease info to grpc header +// https://github.com/containerd/containerd/blob/8459273f806e068e1a6bacfaf1355bbbad738d5e/docs/garbage-collection.md#using-grpc +#[macro_export] +macro_rules! with_lease { + ($req : ident, $ns: expr, $lease_id: expr) => {{ + let mut req = Request::new($req); + let md = req.metadata_mut(); + // https://github.com/containerd/containerd/blob/main/namespaces/grpc.go#L27 + md.insert("containerd-namespace", $ns.parse().unwrap()); + md.insert("containerd-lease", $lease_id.parse().unwrap()); + req + }}; +} + +#[derive(Debug)] +pub(crate) struct LeaseGuard { + pub(crate) lease_id: String, + pub(crate) namespace: String, + pub(crate) address: String, +} + +// Provides a best effort for dropping a lease of the content. If the lease cannot be dropped, it will log a warning +impl Drop for LeaseGuard { + fn drop(&mut self) { + let id = self.lease_id.clone(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let client = rt.block_on(containerd_client::connect(self.address.clone())); + + let channel = match client { + Ok(channel) => channel, + Err(e) => { + log::error!( + "failed to connect to containerd: {}. lease may not be deleted", + e + ); + return; + } + }; + + let mut client = LeasesClient::new(channel); + + rt.block_on(async { + let req = containerd_client::services::v1::DeleteRequest { id, sync: false }; + let req = with_namespace!(req, self.namespace); + let result = client.delete(req).await; + + match result { + Ok(_) => log::debug!("removed lease"), + Err(e) => log::error!("failed to remove lease: {}", e), + } + }); + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs new file mode 100644 index 000000000..58cc8e96e --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs @@ -0,0 +1,4 @@ +mod client; +mod lease; + +pub(crate) use client::Client; diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index 610acd481..315184926 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -1,9 +1,7 @@ use anyhow::{Context, Result}; -use anyhow::{bail}; use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, + Engine, Entrypoint, Instance, RuntimeContext, RuntimeInfo, Stdio, }; -use log::debug; use wasmedge_sdk::config::{ConfigBuilder, HostRegistrationConfigOptions}; use wasmedge_sdk::plugin::PluginManager; use wasmedge_sdk::VmBuilder; diff --git a/crates/containerd-shim-wasmer/src/instance.rs b/crates/containerd-shim-wasmer/src/instance.rs index ea615a880..8517c27c1 100644 --- a/crates/containerd-shim-wasmer/src/instance.rs +++ b/crates/containerd-shim-wasmer/src/instance.rs @@ -1,8 +1,6 @@ use anyhow::Result; -use containerd_shim_wasm::container::{Engine, Entrypoint, Instance, RuntimeContext, Stdio}; -use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, + Engine, Entrypoint, Instance, RuntimeContext, RuntimeInfo, Stdio, }; use wasmer::{Module, Store}; use wasmer_wasix::virtual_fs::host_fs::FileSystem; diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index cc02aaa9b..c823505da 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,11 +1,9 @@ -use std::borrow::Cow; use std::fs::File; use std::sync::OnceLock; use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, PathResolve, RuntimeContext, RuntimeInfo, Source, Stdio, - WasmBinaryType, + Engine, Entrypoint, Instance, RuntimeContext, RuntimeInfo, Stdio, WasmBinaryType, }; use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; @@ -190,7 +188,7 @@ impl WasmtimeEngine { fn execute( &self, - wasm_binary: &Cow<'_, [u8]>, + wasm_binary: &[u8], store: Store, func: String, ) -> Result, anyhow::Error> { From 8cbaf871f77cfb157fe5ef0c238619d23f4f3f1b Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 24 Jan 2024 21:16:05 +0000 Subject: [PATCH 10/19] fix windows compile Signed-off-by: James Sturtevant --- crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs | 2 ++ crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs index 83487c4cc..de1414315 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs @@ -1,3 +1,5 @@ +#![cfg(unix)] + use containerd_client::services::v1::leases_client::LeasesClient; use containerd_client::{tonic, with_namespace}; use tonic::Request; diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs index 58cc8e96e..97c4d7083 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs @@ -1,3 +1,5 @@ +#![cfg(unix)] + mod client; mod lease; From bd70e54ab2c4e88d0d562f968d054531df94d9d3 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 24 Jan 2024 21:47:48 +0000 Subject: [PATCH 11/19] don't fail if cached image was removed Signed-off-by: James Sturtevant --- .../src/sandbox/containerd/client.rs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index 38e4cfeca..b88e363f0 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -395,14 +395,21 @@ impl Client { match image.labels.get(&label) { Some(precompile_digest) if T::can_precompile() => { log::info!("found precompiled module in cache"); - let precompiled = self.read_content(precompile_digest)?; - return Ok(( - vec![WasmLayer { - config: image_config_descriptor.clone(), - layer: precompiled, - }], - platform, - )); + match self.read_content(precompile_digest) { + Ok(precompiled) => { + return Ok(( + vec![WasmLayer { + config: image_config_descriptor.clone(), + layer: precompiled, + }], + platform, + )); + } + Err(e) => { + // log and continue + log::warn!("failed to read precompiled module from cache: {}. Content may have been removed manually, will attempt to recompile", e); + } + } } _ => {} } From 37a83ca6613abf1eecb02e5520d94296302a1813 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 31 Jan 2024 01:10:28 +0000 Subject: [PATCH 12/19] Add some tests Signed-off-by: James Sturtevant --- CONTRIBUTING.md | 6 + Cargo.lock | 5 +- Cargo.toml | 1 + crates/containerd-shim-wasm/Cargo.toml | 4 +- .../src/sandbox/containerd/client.rs | 26 +- crates/containerd-shim-wasm/src/testing.rs | 223 +++++++++++++++++- crates/containerd-shim-wasmtime/src/tests.rs | 76 +++++- crates/oci-tar-builder/src/bin.rs | 5 +- crates/oci-tar-builder/src/lib.rs | 3 + 9 files changed, 333 insertions(+), 16 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fef45d0c7..ddc0154e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,6 +82,12 @@ test instance::wasitest::test_delete_after_create ... ok test instance::wasitest::test_wasi ... ok ``` +Run individual test via cargo adding `RUST_LOG=trace` (adjust the level of logging as needed) to see shim output. Also adjust the test name as needed. + +``` +RUST_LOG=DEBUG cargo test --package containerd-shim-wasmtime --lib -- wasmtime_tests::test_hello_world --exact --nocapture +``` + ### End to End tests The e2e test run on [k3s](https://k3s.io/) and [kind](https://kind.sigs.k8s.io/). A test image is built using [oci-tar-builder](./crates/oci-tar-builder/) and is loaded onto the clusters. This test image is not pushed to an external registry so be sure to use the Makefile targets to build the image and load it on the cluster. diff --git a/Cargo.lock b/Cargo.lock index 438fa31ad..25a538bfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.7" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -628,6 +628,7 @@ dependencies = [ "log", "nix 0.27.1", "oci-spec", + "oci-tar-builder", "prost-types 0.11.9", "protobuf 3.2.0", "serde", diff --git a/Cargo.toml b/Cargo.toml index d2187ae9b..fba93b8d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } containerd-shim = "0.6.0" containerd-shim-wasm = { path = "crates/containerd-shim-wasm", version = "0.4.0" } containerd-shim-wasm-test-modules = { path = "crates/containerd-shim-wasm-test-modules", version = "0.3.1"} +oci-tar-builder = { path = "crates/oci-tar-builder", version = "0.3.1" } crossbeam = { version = "0.8.4", default-features = false } env_logger = "0.10" libc = "0.2.153" diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index ccdd22dee..d4ee65d11 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -16,6 +16,7 @@ anyhow = { workspace = true } chrono = { workspace = true } containerd-shim = { workspace = true } containerd-shim-wasm-test-modules = { workspace = true, optional = true } +oci-tar-builder = { workspace = true, optional = true } crossbeam = { workspace = true } env_logger = { workspace = true, optional = true } git-version = "0.3.9" @@ -54,8 +55,9 @@ ttrpc-codegen = { version = "0.4.2", optional = true } containerd-shim-wasm-test-modules = { workspace = true } env_logger = { workspace = true } tempfile = { workspace = true } +oci-tar-builder = { workspace = true} [features] -testing = ["dep:containerd-shim-wasm-test-modules", "dep:env_logger", "dep:tempfile"] +testing = ["dep:containerd-shim-wasm-test-modules", "dep:env_logger", "dep:tempfile", "dep:oci-tar-builder"] generate_bindings = ["ttrpc-codegen"] generate_doc = [] diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index b88e363f0..11549114f 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -150,11 +150,12 @@ impl Client { let (tx, rx) = mpsc::channel(1); let len = data.len() as i64; - + log::debug!("Writing {} bytes to content store", len); let mut client = ContentClient::new(self.inner.clone()); // Send write request with Stat action to containerd to let it know that we are going to write content // if the content is already there, it will return early with AlreadyExists + log::debug!("Sending stat request to containerd"); let req = WriteContentRequest { r#ref: reference.clone(), action: WriteAction::Stat.into(), @@ -187,6 +188,13 @@ impl Client { )) })?; + // There is a scenario where the content might have been removed manually + // but the content isn't removed from the containerd file system yet. + // In this case if we re-add it at before its removed from file system + // we don't need to copy the content again. Container tells us it found the blob + // by returning the offset of the content that was found. + let data_to_write = data[response.offset as usize..].to_vec(); + let label = precompile_label(info); // Write and commit at same time let mut labels = HashMap::new(); @@ -197,16 +205,20 @@ impl Client { offset: response.offset, expected: expected.clone(), labels, - data, + data: data_to_write, ..Default::default() }; + log::debug!( + "Sending commit request to containerd with response: {:?}", + response + ); tx.send(commit_request) .await - .map_err(|err| ShimError::Containerd(err.to_string()))?; + .map_err(|err| ShimError::Containerd(format!("commit request error: {}", err)))?; let response = response_stream .message() .await - .map_err(|e| ShimError::Containerd(e.to_string()))? + .map_err(|err| ShimError::Containerd(format!("response stream error: {}", err)))? .ok_or_else(|| { ShimError::Containerd(format!( "no response received after write request for {}", @@ -214,6 +226,7 @@ impl Client { )) })?; + log::debug!("Validating response"); // client should validate that all bytes were written and that the digest matches if response.offset != len { return Err(ShimError::Containerd(format!( @@ -312,6 +325,7 @@ impl Client { paths: vec!["labels".to_string()], }), }; + let req = with_namespace!(req, self.namespace); let image = ImagesClient::new(self.inner.clone()) .update(req) @@ -394,9 +408,10 @@ impl Client { let label = precompile_label(T::info()); match image.labels.get(&label) { Some(precompile_digest) if T::can_precompile() => { - log::info!("found precompiled module in cache"); + log::info!("found precompiled label: {} ", &label); match self.read_content(precompile_digest) { Ok(precompiled) => { + log::info!("found precompiled module in cache: {} ", &precompile_digest); return Ok(( vec![WasmLayer { config: image_config_descriptor.clone(), @@ -429,6 +444,7 @@ impl Client { if T::can_precompile() { log::info!("precompiling module"); let precompiled = engine.precompile(layers.as_slice())?; + log::info!("precompiling module: {}", image_digest.clone()); let precompiled_content = self.save_content(precompiled.clone(), image_digest.clone(), T::info())?; diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index 348668b26..2986b4a95 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -1,22 +1,28 @@ //! Testing utilities used across different modules use std::collections::HashMap; -use std::fs::{create_dir, read_to_string, write, File}; +use std::fs::{self, create_dir, read_to_string, write, File}; use std::marker::PhantomData; use std::ops::Add; +use std::process::Command; use std::time::Duration; use anyhow::{bail, Result}; pub use containerd_shim_wasm_test_modules as modules; +use oci_spec::image::{self as spec, Arch}; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; +use oci_tar_builder::{Builder, WASM_LAYER_MEDIA_TYPE}; use crate::sandbox::{Instance, InstanceConfig}; use crate::sys::signals::SIGKILL; +const TEST_NAMESPACE: &str = "runwasi-test"; + pub struct WasiTestBuilder where WasiInstance::Engine: Default + Send + Sync + Clone, { + container_name: String, tempdir: tempfile::TempDir, _phantom: PhantomData, } @@ -55,6 +61,7 @@ where write(dir.join("stderr"), "")?; let builder = Self { + container_name: "test".to_string(), tempdir, _phantom: Default::default(), } @@ -114,6 +121,85 @@ where Ok(self) } + pub fn as_oci_image( + mut self, + image_name: Option, + container_name: Option, + ) -> Result<(Self, oci_helpers::OCICleanup)> { + let mut builder = Builder::default(); + + let dir = self.tempdir.path(); + let wasm_path = dir.join("rootfs").join("hello.wasm"); + builder.add_layer_with_media_type(&wasm_path, WASM_LAYER_MEDIA_TYPE.to_string()); + + let config = spec::ConfigBuilder::default() + .entrypoint(vec!["_start".to_string()]) + .build() + .unwrap(); + + let img = spec::ImageConfigurationBuilder::default() + .config(config) + .os("wasip1") + .architecture(Arch::Wasm) + .rootfs( + spec::RootFsBuilder::default() + .diff_ids(vec![]) + .build() + .unwrap(), + ) + .build()?; + + let image_name = image_name.unwrap_or("localhost/hello:latest".to_string()); + builder.add_config(img, image_name.clone()); + + let img = dir.join("img.tar"); + let f = File::create(img.clone())?; + builder.build(f)?; + + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("image") + .arg("import") + .arg("--all-platforms") + .arg(img) + .spawn()? + .wait()? + .success(); + + if !success { + // if the container still exists try cleaning it up + bail!(" failed to import image"); + } + + fs::remove_file(&wasm_path)?; + + let container_name = container_name.unwrap_or("test".to_string()); + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("c") + .arg("create") + .arg(&image_name) + .arg(&container_name) + .spawn()? + .wait()? + .success(); + + if !success { + bail!(" failed to create container for image"); + } + + self.container_name = container_name.clone(); + Ok(( + self, + oci_helpers::OCICleanup { + image_name, + container_name, + }, + )) + } + pub fn build(self) -> Result> { let tempdir = self.tempdir; let dir = tempdir.path(); @@ -122,7 +208,7 @@ where let mut cfg = InstanceConfig::new( WasiInstance::Engine::default(), - "test_namespace", + TEST_NAMESPACE, "/run/containerd/containerd.sock", ); cfg.set_bundle(dir) @@ -130,7 +216,7 @@ where .set_stderr(dir.join("stderr")) .set_stdin(dir.join("stdin")); - let instance = WasiInstance::new("test".to_string(), Some(&cfg))?; + let instance = WasiInstance::new(self.container_name, Some(&cfg))?; Ok(WasiTest { instance, tempdir }) } } @@ -181,3 +267,134 @@ where Ok((status, stdout, stderr)) } } + +pub mod oci_helpers { + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + use anyhow::{bail, Result}; + + use super::TEST_NAMESPACE; + + pub struct OCICleanup { + pub image_name: String, + pub container_name: String, + } + + impl Drop for OCICleanup { + fn drop(&mut self) { + log::trace!("dropping OCIGuard"); + clean_image(self.image_name.clone()).unwrap(); + clean_container(self.container_name.clone()).unwrap(); + } + } + + pub fn clean_container(container_name: String) -> Result<()> { + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("c") + .arg("rm") + .arg(container_name) + .spawn()? + .wait()? + .success(); + + if !success { + bail!("failed to clean container") + } + + Ok(()) + } + + pub fn clean_image(image_name: String) -> Result<()> { + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("i") + .arg("rm") + .arg(image_name) + .spawn()? + .wait()? + .success(); + + if !success { + bail!("failed to clean image"); + } + + // the content isn't removed immediately, so we need to wait for it to be removed + // otherwise the next test will not behave as expected + let start = Instant::now(); + let timeout = Duration::from_secs(300); + loop { + let output = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("content") + .arg("ls") + .arg("-q") + .output()?; + + if output.stdout.is_empty() { + break; + } + + if start.elapsed() > timeout { + bail!("timed out waiting for content to be removed"); + } + + log::debug!("waiting for content to be removed"); + } + + Ok(()) + } + + pub fn get_image_label() -> Result<(String, String)> { + let mut grep = Command::new("grep") + .arg("-ohE") + .arg("runwasi.io/precompiled/wasmtime/0.3.1=.*") + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .spawn()?; + + Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("i") + .arg("ls") + .stdout(grep.stdin.take().unwrap()) + .spawn()?; + + let output = grep.wait_with_output()?; + + let stdout = String::from_utf8(output.stdout)?; + + log::info!("stdout: {}", stdout); + + let label: Vec<&str> = stdout.split('=').collect(); + + Ok(( + label.first().unwrap().trim().to_string(), + label.last().unwrap().trim().to_string(), + )) + } + + pub fn remove_content(digest: String) -> Result<()> { + log::info!("cleaning content '{}'", digest); + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("content") + .arg("rm") + .arg(digest) + .spawn()? + .wait()? + .success(); + + if !success { + bail!("failed to remove content"); + } + + Ok(()) + } +} diff --git a/crates/containerd-shim-wasmtime/src/tests.rs b/crates/containerd-shim-wasmtime/src/tests.rs index 1073880ad..adcd41eb2 100644 --- a/crates/containerd-shim-wasmtime/src/tests.rs +++ b/crates/containerd-shim-wasmtime/src/tests.rs @@ -2,7 +2,7 @@ use std::time::Duration; //use containerd_shim_wasm::sandbox::Instance; use containerd_shim_wasm::testing::modules::*; -use containerd_shim_wasm::testing::WasiTest; +use containerd_shim_wasm::testing::{oci_helpers, WasiTest}; use serial_test::serial; use crate::instance::WasmtimeInstance as WasiInstance; @@ -29,6 +29,80 @@ fn test_hello_world() -> anyhow::Result<()> { Ok(()) } +#[test] +#[serial] +fn test_hello_world_oci() -> anyhow::Result<()> { + let (builder, _oci_cleanup) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, None)?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + +#[test] +#[serial] +fn test_hello_world_oci_uses_precompiled() -> anyhow::Result<()> { + let (builder, _oci_cleanup1) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c1".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + let (label, _id) = oci_helpers::get_image_label()?; + assert!(label.starts_with("runwasi.io/precompiled/wasmtime/")); + + // run second time, it should succeed without recompiling + let (builder, _oci_cleanup2) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c2".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + +#[test] +#[serial] +fn test_hello_world_oci_uses_precompiled_when_content_removed() -> anyhow::Result<()> { + let (builder, _oci_cleanup1) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c1".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + let (label, id) = oci_helpers::get_image_label()?; + + // remove the compiled content from the cache + assert!(label.starts_with("runwasi.io/precompiled/wasmtime/")); + oci_helpers::remove_content(id)?; + + // run second time, it should succeed + let (builder, _oci_cleanup2) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c2".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { diff --git a/crates/oci-tar-builder/src/bin.rs b/crates/oci-tar-builder/src/bin.rs index 0d1ba80b8..d21e88aa1 100644 --- a/crates/oci-tar-builder/src/bin.rs +++ b/crates/oci-tar-builder/src/bin.rs @@ -5,10 +5,7 @@ use std::{env, fs}; use anyhow::Context; use clap::Parser; use oci_spec::image::{self as spec, Arch}; -use oci_tar_builder::Builder; - -pub const WASM_LAYER_MEDIA_TYPE: &str = - "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; +use oci_tar_builder::{Builder, WASM_LAYER_MEDIA_TYPE}; pub fn main() { let args = Args::parse(); diff --git a/crates/oci-tar-builder/src/lib.rs b/crates/oci-tar-builder/src/lib.rs index cabf785a1..b4f50e36e 100644 --- a/crates/oci-tar-builder/src/lib.rs +++ b/crates/oci-tar-builder/src/lib.rs @@ -41,6 +41,9 @@ struct DockerManifest { layers: Vec, } +pub const WASM_LAYER_MEDIA_TYPE: &str = + "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; + impl Builder { pub fn add_config(&mut self, config: ImageConfiguration, name: String) -> &mut Self { self.configs.push((config, name)); From b49594e6504f04e3419b7a4e2a055776119c8078 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 31 Jan 2024 01:16:43 +0000 Subject: [PATCH 13/19] Add basic oci tests to other runtimes Signed-off-by: James Sturtevant --- crates/containerd-shim-wasmedge/src/tests.rs | 15 +++++++++++++++ crates/containerd-shim-wasmer/src/tests.rs | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/crates/containerd-shim-wasmedge/src/tests.rs b/crates/containerd-shim-wasmedge/src/tests.rs index e292f4bd5..050ec3267 100644 --- a/crates/containerd-shim-wasmedge/src/tests.rs +++ b/crates/containerd-shim-wasmedge/src/tests.rs @@ -29,6 +29,21 @@ fn test_hello_world() -> anyhow::Result<()> { Ok(()) } +#[test] +#[serial] +fn test_hello_world_oci() -> anyhow::Result<()> { + let (builder, _oci_cleanup) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, None)?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { diff --git a/crates/containerd-shim-wasmer/src/tests.rs b/crates/containerd-shim-wasmer/src/tests.rs index 15ee5b478..7bb6131da 100644 --- a/crates/containerd-shim-wasmer/src/tests.rs +++ b/crates/containerd-shim-wasmer/src/tests.rs @@ -29,6 +29,22 @@ fn test_hello_world() -> anyhow::Result<()> { Ok(()) } +#[test] +#[serial] +fn test_hello_world_oci() -> anyhow::Result<()> { + let (builder, _oci_cleanup) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, None)?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + + #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { From 91a8da390ca48595c7c3d322037053b7ed4bd583 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 30 Jan 2024 17:19:53 -0800 Subject: [PATCH 14/19] Apply suggestions from code review Co-authored-by: Vaughn Dice Signed-off-by: James Sturtevant Signed-off-by: James Sturtevant --- crates/containerd-shim-wasm/src/container/engine.rs | 4 ++-- crates/containerd-shim-wasm/src/sandbox/containerd/client.rs | 2 +- crates/containerd-shim-wasmer/src/tests.rs | 1 - crates/containerd-shim-wasmtime/src/instance.rs | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 054f88664..1998dd6d9 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -60,7 +60,7 @@ pub trait Engine: Clone + Send + Sync + 'static { &["application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"] } - /// Precomiples a module that in in the WASM OCI layer format + /// Precompiles a module that is in the WASM OCI layer format /// This is used to precompile a module before it is run and will be called if can_precompile returns true. /// It is called only the first time a module is run and the resulting bytes will be cached in the containerd content store. /// The cached, precompiled module will be reloaded on subsequent runs. @@ -68,7 +68,7 @@ pub trait Engine: Clone + Send + Sync + 'static { bail!("precompilation not supported for this runtime") } - /// Precomiple a module + /// Precompile a module fn can_precompile() -> bool { false } diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index 11549114f..c208d9514 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -455,7 +455,7 @@ impl Client { self.update_image(image)?; // The original image is considered a root object, by adding a ref to the new compiled content - // We tell to containerd to not garbage collect the new content until this image is removed from the system + // We tell containerd to not garbage collect the new content until this image is removed from the system // this ensures that we keep the content around after the lease is dropped log::debug!("updating content with precompile digest to avoid garbage collection"); let mut image_content = self.get_info(image_digest.clone())?; diff --git a/crates/containerd-shim-wasmer/src/tests.rs b/crates/containerd-shim-wasmer/src/tests.rs index 7bb6131da..591a4c5c0 100644 --- a/crates/containerd-shim-wasmer/src/tests.rs +++ b/crates/containerd-shim-wasmer/src/tests.rs @@ -44,7 +44,6 @@ fn test_hello_world_oci() -> anyhow::Result<()> { Ok(()) } - #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index c823505da..2e099b0cc 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -105,7 +105,7 @@ impl Engine for WasmtimeEngine { fn precompile(&self, layers: &[Vec]) -> Result> { match layers { [layer] => self.engine.precompile_module(layer), - _ => bail!("only a single module is supported when when precompiling"), + _ => bail!("only a single module is supported when precompiling"), } } @@ -209,7 +209,7 @@ impl WasmtimeEngine { self.execute_module(module, store, &func) } Some(Precompiled::Component) => { - log::info!("using precompiled module"); + log::info!("using precompiled component"); let component = unsafe { Component::deserialize(&self.engine, wasm_binary) }?; self.execute_component(component, store, func) } From 826d6c5694a43e489b22b1b34f93adff8fa5ca94 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 1 Feb 2024 18:41:49 +0000 Subject: [PATCH 15/19] Add some debuging support Signed-off-by: James Sturtevant --- crates/containerd-shim-wasm/src/container/context.rs | 1 + .../src/sandbox/containerd/client.rs | 2 +- crates/containerd-shim-wasm/src/sandbox/oci.rs | 2 +- .../src/sys/unix/container/instance.rs | 2 +- crates/containerd-shim-wasm/src/testing.rs | 10 ++++++---- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/containerd-shim-wasm/src/container/context.rs b/crates/containerd-shim-wasm/src/container/context.rs index 5f4475a5b..e5b6881c8 100644 --- a/crates/containerd-shim-wasm/src/container/context.rs +++ b/crates/containerd-shim-wasm/src/container/context.rs @@ -34,6 +34,7 @@ pub trait RuntimeContext { } /// The source for a WASI module / components. +#[derive(Debug)] pub enum Source<'a> { // The WASI module is a file in the file system. File(PathBuf), diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index c208d9514..e4c599656 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -382,7 +382,7 @@ impl Client { pub fn load_modules( &self, containerd_id: impl ToString, - engine: T, + engine: &T, ) -> Result<(Vec, Platform)> { let container = self.get_container(containerd_id.to_string())?; let mut image = self.get_image(container.image)?; diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index c7edfcf55..0d3478007 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -11,7 +11,7 @@ use oci_spec::image::Descriptor; use super::error::Result; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct WasmLayer { pub config: Descriptor, pub layer: Vec, diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index d297b6b46..8ab0435e9 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -45,7 +45,7 @@ impl SandboxInstance for Instance { // check if container is OCI image with wasm layers and attempt to read the module let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address().as_str(), &namespace)? - .load_modules(&id, engine.clone()) + .load_modules(&id, &engine) .unwrap_or_else(|e| { log::warn!("Error obtaining wasm layers for container {id}. Will attempt to use files inside container image. Error: {e}"); (vec![], Platform::default()) diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index 2986b4a95..a1602c657 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -283,13 +283,14 @@ pub mod oci_helpers { impl Drop for OCICleanup { fn drop(&mut self) { - log::trace!("dropping OCIGuard"); - clean_image(self.image_name.clone()).unwrap(); + log::debug!("dropping OCIGuard"); clean_container(self.container_name.clone()).unwrap(); + clean_image(self.image_name.clone()).unwrap(); } } pub fn clean_container(container_name: String) -> Result<()> { + log::debug!("deleting container '{}'", container_name); let success = Command::new("ctr") .arg("-n") .arg(TEST_NAMESPACE) @@ -308,6 +309,7 @@ pub mod oci_helpers { } pub fn clean_image(image_name: String) -> Result<()> { + log::debug!("deleting image '{}'", image_name); let success = Command::new("ctr") .arg("-n") .arg(TEST_NAMESPACE) @@ -343,7 +345,7 @@ pub mod oci_helpers { bail!("timed out waiting for content to be removed"); } - log::debug!("waiting for content to be removed"); + log::trace!("waiting for content to be removed"); } Ok(()) @@ -380,7 +382,7 @@ pub mod oci_helpers { } pub fn remove_content(digest: String) -> Result<()> { - log::info!("cleaning content '{}'", digest); + log::debug!("cleaning content '{}'", digest); let success = Command::new("ctr") .arg("-n") .arg(TEST_NAMESPACE) From 261afbcf2a82e5238904bc2c71d12a16ca23dba5 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 1 Feb 2024 19:52:19 +0000 Subject: [PATCH 16/19] rebase Signed-off-by: James Sturtevant --- crates/containerd-shim-wasmtime/src/instance.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 2e099b0cc..22b124c57 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -8,7 +8,7 @@ use containerd_shim_wasm::container::{ use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; use wasmtime::{Module, Precompiled, Store}; -use wasmtime_wasi::preview2::{self as wasi_preview2, Table}; +use wasmtime_wasi::preview2::{self as wasi_preview2}; use wasmtime_wasi::{self as wasi_preview1, Dir}; pub type WasmtimeInstance = Instance; From 603d4487ea9103774eafc355dd3abd539066d5ad Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 2 Feb 2024 01:28:56 +0000 Subject: [PATCH 17/19] (hack) make tests pass Signed-off-by: James Sturtevant --- crates/containerd-shim-wasmtime/src/instance.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 22b124c57..83624a8b3 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -21,6 +21,7 @@ pub struct WasmtimeEngine { impl Default for WasmtimeEngine { fn default() -> Self { let mut config = wasmtime::Config::new(); + config.parallel_compilation(false); config.wasm_component_model(true); // enable component linking Self { engine: wasmtime::Engine::new(&config) From 063242a20296997112c87ea6d1c9955eea157afc Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 6 Feb 2024 18:23:09 +0000 Subject: [PATCH 18/19] Use precompilation hash and test default config Signed-off-by: James Sturtevant --- .../src/container/engine.rs | 25 ++++---- .../containerd-shim-wasm/src/container/mod.rs | 2 +- .../src/container/tests.rs | 8 +-- .../src/sandbox/containerd/client.rs | 59 +++++++------------ .../src/sys/unix/container/executor.rs | 4 +- .../src/sys/unix/container/instance.rs | 2 +- crates/containerd-shim-wasm/src/testing.rs | 2 +- .../containerd-shim-wasmedge/src/instance.rs | 11 +--- crates/containerd-shim-wasmer/src/instance.rs | 11 +--- .../containerd-shim-wasmtime/src/instance.rs | 57 +++++++++++------- crates/containerd-shim-wasmtime/src/tests.rs | 35 +++++++++-- 11 files changed, 113 insertions(+), 103 deletions(-) diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 1998dd6d9..f3461f6c8 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -7,16 +7,9 @@ use super::Source; use crate::container::{PathResolve, RuntimeContext}; use crate::sandbox::Stdio; -/// RuntimeInfo contains the name and version of the runtime that is running -#[derive(Clone, Debug, Default)] -pub struct RuntimeInfo { - pub name: &'static str, - pub version: &'static str, -} - pub trait Engine: Clone + Send + Sync + 'static { /// The name to use for this engine - fn info() -> &'static RuntimeInfo; + fn name() -> &'static str; /// Run a WebAssembly container fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result; @@ -68,8 +61,18 @@ pub trait Engine: Clone + Send + Sync + 'static { bail!("precompilation not supported for this runtime") } - /// Precompile a module - fn can_precompile() -> bool { - false + /// Can_precompile lets the shim know if the runtime supports precompilation. + /// When it returns Some(unique_string) the `unique_string` will be used as a cache key for the precompiled module. + /// + /// `unique_string` should at least include the version of the shim running but could include other information such as a hash + /// of the version and cpu type and other important information in the validation of being able to use precompiled module. + /// If the string doesn't match then the module will be recompiled and cached with the new `unique_string`. + /// + /// This string will be used in the following way: + /// "runwasi.io/precompiled//" + /// + /// When it returns None the runtime will not be asked to precompile the module. This is the default value. + fn can_precompile(&self) -> Option { + None } } diff --git a/crates/containerd-shim-wasm/src/container/mod.rs b/crates/containerd-shim-wasm/src/container/mod.rs index a58c4d5a8..64e58870b 100644 --- a/crates/containerd-shim-wasm/src/container/mod.rs +++ b/crates/containerd-shim-wasm/src/container/mod.rs @@ -17,7 +17,7 @@ mod wasm; pub(crate) use context::WasiContext; pub use context::{Entrypoint, RuntimeContext, Source}; -pub use engine::{Engine, RuntimeInfo}; +pub use engine::Engine; pub use instance::Instance; pub use path::PathResolve; pub use wasm::WasmBinaryType; diff --git a/crates/containerd-shim-wasm/src/container/tests.rs b/crates/containerd-shim-wasm/src/container/tests.rs index 84080d69e..bbf83869b 100644 --- a/crates/containerd-shim-wasm/src/container/tests.rs +++ b/crates/containerd-shim-wasm/src/container/tests.rs @@ -1,6 +1,5 @@ use anyhow::bail; -use super::engine::RuntimeInfo; use crate::container::{Engine, RuntimeContext, Stdio}; use crate::sys::container::instance::Instance; use crate::testing::WasiTest; @@ -9,11 +8,8 @@ use crate::testing::WasiTest; struct EngineFailingValidation; impl Engine for EngineFailingValidation { - fn info() -> &'static RuntimeInfo { - &RuntimeInfo { - name: "wasi_instance", - version: "0.0.0", - } + fn name() -> &'static str { + "wasi_instance" } fn can_handle(&self, _ctx: &impl RuntimeContext) -> anyhow::Result<()> { bail!("can't handle"); diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs index e4c599656..37e83f28e 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -25,7 +25,7 @@ use tokio_stream::wrappers::ReceiverStream; use tonic::{Code, Request}; use super::lease::LeaseGuard; -use crate::container::{Engine, RuntimeInfo}; +use crate::container::Engine; use crate::sandbox::error::{Error as ShimError, Result}; use crate::sandbox::oci::{self, WasmLayer}; use crate::with_lease; @@ -139,10 +139,10 @@ impl Client { &self, data: Vec, original_digest: String, - info: &RuntimeInfo, + label: &str, ) -> Result { let expected = format!("sha256:{}", digest(data.clone())); - let reference = format!("precompile-{}-{}-{}", info.name, info.version, expected); + let reference = format!("precompile-{}", label); let lease = self.lease(reference.clone())?; let digest = self.rt.block_on(async { @@ -195,10 +195,9 @@ impl Client { // by returning the offset of the content that was found. let data_to_write = data[response.offset as usize..].to_vec(); - let label = precompile_label(info); // Write and commit at same time let mut labels = HashMap::new(); - labels.insert(label, original_digest.clone()); + labels.insert(label.to_string(), original_digest.clone()); let commit_request = WriteContentRequest { action: WriteAction::Commit.into(), total: len, @@ -405,10 +404,14 @@ impl Client { log::info!("found manifest with WASM OCI image format."); // This label is unique across runtimes and version of the shim running // a precompiled component/module will not work across different runtimes or versions - let label = precompile_label(T::info()); - match image.labels.get(&label) { - Some(precompile_digest) if T::can_precompile() => { - log::info!("found precompiled label: {} ", &label); + let (can_precompile, precompile_id) = match engine.can_precompile() { + Some(precompile_id) => (true, precompile_label(T::name(), &precompile_id)), + None => (false, "".to_string()), + }; + + match image.labels.get(&precompile_id) { + Some(precompile_digest) if can_precompile => { + log::info!("found precompiled label: {} ", &precompile_id); match self.read_content(precompile_digest) { Ok(precompiled) => { log::info!("found precompiled module in cache: {} ", &precompile_digest); @@ -441,17 +444,17 @@ impl Client { return Ok((vec![], platform)); } - if T::can_precompile() { + if can_precompile { log::info!("precompiling module"); let precompiled = engine.precompile(layers.as_slice())?; log::info!("precompiling module: {}", image_digest.clone()); let precompiled_content = - self.save_content(precompiled.clone(), image_digest.clone(), T::info())?; + self.save_content(precompiled.clone(), image_digest.clone(), &precompile_id)?; log::debug!("updating image with compiled content digest"); image .labels - .insert(label, precompiled_content.digest.clone()); + .insert(precompile_id, precompiled_content.digest.clone()); self.update_image(image)?; // The original image is considered a root object, by adding a ref to the new compiled content @@ -486,8 +489,8 @@ impl Client { } } -fn precompile_label(info: &RuntimeInfo) -> String { - format!("{}/{}/{}", PRECOMPILE_PREFIX, info.name, info.version) +fn precompile_label(name: &str, version: &str) -> String { + format!("{}/{}/{}", PRECOMPILE_PREFIX, name, version) } fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { @@ -510,15 +513,9 @@ mod tests { let expected = digest(data.clone()); let expected = format!("sha256:{}", expected); + let label = precompile_label("test", "hasdfh"); let returned = client - .save_content( - data, - "original".to_string(), - &RuntimeInfo { - name: "test", - version: "0.0.0", - }, - ) + .save_content(data, "original".to_string(), &label) .unwrap(); assert_eq!(expected, returned.digest.clone()); @@ -526,14 +523,7 @@ mod tests { assert_eq!(data, b"hello world"); client - .save_content( - data.clone(), - "original".to_string(), - &RuntimeInfo { - name: "test", - version: "0.0.0", - }, - ) + .save_content(data.clone(), "original".to_string(), &label) .expect_err("Should not be able to save when lease is open"); // need to drop the lease to be able to create a second one @@ -542,14 +532,7 @@ mod tests { // a second call should be successful since it already exists let returned = client - .save_content( - data, - "original".to_string(), - &RuntimeInfo { - name: "test", - version: "0.0.0", - }, - ) + .save_content(data, "original".to_string(), &label) .unwrap(); assert_eq!(expected, returned.digest); diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs index 2c943eccb..81e45933e 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs @@ -36,7 +36,7 @@ impl LibcontainerExecutor for Executor { fn validate(&self, spec: &Spec) -> Result<(), ExecutorValidationError> { // We can handle linux container. We delegate wasm container to the engine. match self.inner(spec) { - InnerExecutor::CantHandle => Err(ExecutorValidationError::CantHandle(E::info().name)), + InnerExecutor::CantHandle => Err(ExecutorValidationError::CantHandle(E::name())), _ => Ok(()), } } @@ -45,7 +45,7 @@ impl LibcontainerExecutor for Executor { // If it looks like a linux container, run it as a linux container. // Otherwise, run it as a wasm container match self.inner(spec) { - InnerExecutor::CantHandle => Err(LibcontainerExecutorError::CantHandle(E::info().name)), + InnerExecutor::CantHandle => Err(LibcontainerExecutorError::CantHandle(E::name())), InnerExecutor::Linux => { log::info!("executing linux container"); self.stdio.take().redirect().unwrap(); diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index 8ab0435e9..628be4ef8 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -39,7 +39,7 @@ impl SandboxInstance for Instance { let engine = cfg.get_engine(); let bundle = cfg.get_bundle().to_path_buf(); let namespace = cfg.get_namespace(); - let rootdir = Path::new(DEFAULT_CONTAINER_ROOT_DIR).join(E::info().name); + let rootdir = Path::new(DEFAULT_CONTAINER_ROOT_DIR).join(E::name()); let rootdir = determine_rootdir(&bundle, &namespace, rootdir)?; let stdio = Stdio::init_from_cfg(cfg)?; diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index a1602c657..1c1131386 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -354,7 +354,7 @@ pub mod oci_helpers { pub fn get_image_label() -> Result<(String, String)> { let mut grep = Command::new("grep") .arg("-ohE") - .arg("runwasi.io/precompiled/wasmtime/0.3.1=.*") + .arg("runwasi.io/precompiled/.*") .stdout(Stdio::piped()) .stdin(Stdio::piped()) .spawn()?; diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index 315184926..137c35f66 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -1,7 +1,5 @@ use anyhow::{Context, Result}; -use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, RuntimeContext, RuntimeInfo, Stdio, -}; +use containerd_shim_wasm::container::{Engine, Entrypoint, Instance, RuntimeContext, Stdio}; use wasmedge_sdk::config::{ConfigBuilder, HostRegistrationConfigOptions}; use wasmedge_sdk::plugin::PluginManager; use wasmedge_sdk::VmBuilder; @@ -27,11 +25,8 @@ impl Default for WasmEdgeEngine { } impl Engine for WasmEdgeEngine { - fn info() -> &'static RuntimeInfo { - &RuntimeInfo { - name: "wasmedge", - version: env!("CARGO_PKG_VERSION"), - } + fn name() -> &'static str { + "wasmedge" } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { diff --git a/crates/containerd-shim-wasmer/src/instance.rs b/crates/containerd-shim-wasmer/src/instance.rs index 8517c27c1..34fdb0104 100644 --- a/crates/containerd-shim-wasmer/src/instance.rs +++ b/crates/containerd-shim-wasmer/src/instance.rs @@ -1,7 +1,5 @@ use anyhow::Result; -use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, RuntimeContext, RuntimeInfo, Stdio, -}; +use containerd_shim_wasm::container::{Engine, Entrypoint, Instance, RuntimeContext, Stdio}; use wasmer::{Module, Store}; use wasmer_wasix::virtual_fs::host_fs::FileSystem; use wasmer_wasix::{WasiEnv, WasiError}; @@ -14,11 +12,8 @@ pub struct WasmerEngine { } impl Engine for WasmerEngine { - fn info() -> &'static RuntimeInfo { - &RuntimeInfo { - name: "wasmer", - version: env!("CARGO_PKG_VERSION"), - } + fn name() -> &'static str { + "wasmer" } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 83624a8b3..05dcb406e 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,32 +1,49 @@ +use std::collections::hash_map::DefaultHasher; use std::fs::File; -use std::sync::OnceLock; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ - Engine, Entrypoint, Instance, RuntimeContext, RuntimeInfo, Stdio, WasmBinaryType, + Engine, Entrypoint, Instance, RuntimeContext, Stdio, WasmBinaryType, }; use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; -use wasmtime::{Module, Precompiled, Store}; +use wasmtime::{Config, Module, Precompiled, Store}; use wasmtime_wasi::preview2::{self as wasi_preview2}; use wasmtime_wasi::{self as wasi_preview1, Dir}; -pub type WasmtimeInstance = Instance; +pub type WasmtimeInstance = Instance>; #[derive(Clone)] -pub struct WasmtimeEngine { +pub struct WasmtimeEngine { engine: wasmtime::Engine, + config_type: PhantomData, } -impl Default for WasmtimeEngine { - fn default() -> Self { +#[derive(Clone)] +pub struct DefaultConfig {} + +impl WasiConfig for DefaultConfig { + fn new_config() -> Config { let mut config = wasmtime::Config::new(); - config.parallel_compilation(false); config.wasm_component_model(true); // enable component linking + config + } +} + +pub trait WasiConfig { + fn new_config() -> Config; +} + +impl Default for WasmtimeEngine { + fn default() -> Self { + let config = T::new_config(); Self { engine: wasmtime::Engine::new(&config) .context("failed to create wasmtime engine") .unwrap(), + config_type: PhantomData, } } } @@ -57,17 +74,9 @@ impl wasmtime_wasi::preview2::WasiView for WasiCtx { } } -fn version_info() -> &'static RuntimeInfo { - static INFO: OnceLock = OnceLock::new(); - INFO.get_or_init(|| RuntimeInfo { - name: "wasmtime", - version: option_env!("WASMTIME_VERSION_INFO").unwrap_or(env!("CARGO_PKG_VERSION")), - }) -} - -impl Engine for WasmtimeEngine { - fn info() -> &'static RuntimeInfo { - version_info() +impl Engine for WasmtimeEngine { + fn name() -> &'static str { + "wasmtime" } fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { @@ -110,12 +119,16 @@ impl Engine for WasmtimeEngine { } } - fn can_precompile() -> bool { - true + fn can_precompile(&self) -> Option { + let mut hasher = DefaultHasher::new(); + self.engine + .precompile_compatibility_hash() + .hash(&mut hasher); + Some(hasher.finish().to_string()) } } -impl WasmtimeEngine { +impl WasmtimeEngine { /// Execute a wasm module. /// /// This function adds wasi_preview1 to the linker and can be utilized diff --git a/crates/containerd-shim-wasmtime/src/tests.rs b/crates/containerd-shim-wasmtime/src/tests.rs index adcd41eb2..8a49bea6e 100644 --- a/crates/containerd-shim-wasmtime/src/tests.rs +++ b/crates/containerd-shim-wasmtime/src/tests.rs @@ -1,11 +1,31 @@ use std::time::Duration; -//use containerd_shim_wasm::sandbox::Instance; +use containerd_shim_wasm::container::Instance; use containerd_shim_wasm::testing::modules::*; use containerd_shim_wasm::testing::{oci_helpers, WasiTest}; use serial_test::serial; - -use crate::instance::WasmtimeInstance as WasiInstance; +use wasmtime::Config; +use WasmtimeTestInstance as WasiInstance; + +use crate::instance::{WasiConfig, WasmtimeEngine}; + +// use test configuration to avoid dead locks when running tests +type WasmtimeTestInstance = Instance>; + +#[derive(Clone)] +struct WasiTestConfig {} + +impl WasiConfig for WasiTestConfig { + fn new_config() -> Config { + let mut config = wasmtime::Config::new(); + // Disable Wasmtime parallel compilation for the + // https://github.com/containerd/runwasi/issues/357 + // see https://github.com/containerd/runwasi/pull/405#issuecomment-1928468714 for details + config.parallel_compilation(false); + config.wasm_component_model(true); // enable component linking + config + } +} #[test] #[serial] @@ -56,8 +76,13 @@ fn test_hello_world_oci_uses_precompiled() -> anyhow::Result<()> { assert_eq!(exit_code, 0); assert_eq!(stdout, "hello world\n"); - let (label, _id) = oci_helpers::get_image_label()?; - assert!(label.starts_with("runwasi.io/precompiled/wasmtime/")); + let (label, id) = oci_helpers::get_image_label()?; + assert!( + label.starts_with("runwasi.io/precompiled/wasmtime/"), + "was {}={}", + label, + id + ); // run second time, it should succeed without recompiling let (builder, _oci_cleanup2) = WasiTest::::builder()? From 15a82127e2d8d7da9e8dc8e5a100f944d34b0c31 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 6 Feb 2024 20:38:20 +0000 Subject: [PATCH 19/19] Simplify trait Signed-off-by: James Sturtevant --- crates/containerd-shim-wasmtime/src/instance.rs | 8 ++++---- crates/containerd-shim-wasmtime/src/tests.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 05dcb406e..60585c818 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -16,7 +16,7 @@ use wasmtime_wasi::{self as wasi_preview1, Dir}; pub type WasmtimeInstance = Instance>; #[derive(Clone)] -pub struct WasmtimeEngine { +pub struct WasmtimeEngine { engine: wasmtime::Engine, config_type: PhantomData, } @@ -32,11 +32,11 @@ impl WasiConfig for DefaultConfig { } } -pub trait WasiConfig { +pub trait WasiConfig: Clone + Sync + Send + 'static { fn new_config() -> Config; } -impl Default for WasmtimeEngine { +impl Default for WasmtimeEngine { fn default() -> Self { let config = T::new_config(); Self { @@ -74,7 +74,7 @@ impl wasmtime_wasi::preview2::WasiView for WasiCtx { } } -impl Engine for WasmtimeEngine { +impl Engine for WasmtimeEngine { fn name() -> &'static str { "wasmtime" } diff --git a/crates/containerd-shim-wasmtime/src/tests.rs b/crates/containerd-shim-wasmtime/src/tests.rs index 8a49bea6e..fcab3a50a 100644 --- a/crates/containerd-shim-wasmtime/src/tests.rs +++ b/crates/containerd-shim-wasmtime/src/tests.rs @@ -10,6 +10,7 @@ use WasmtimeTestInstance as WasiInstance; use crate::instance::{WasiConfig, WasmtimeEngine}; // use test configuration to avoid dead locks when running tests +// https://github.com/containerd/runwasi/issues/357 type WasmtimeTestInstance = Instance>; #[derive(Clone)] @@ -18,8 +19,7 @@ struct WasiTestConfig {} impl WasiConfig for WasiTestConfig { fn new_config() -> Config { let mut config = wasmtime::Config::new(); - // Disable Wasmtime parallel compilation for the - // https://github.com/containerd/runwasi/issues/357 + // Disable Wasmtime parallel compilation for the tests // see https://github.com/containerd/runwasi/pull/405#issuecomment-1928468714 for details config.parallel_compilation(false); config.wasm_component_model(true); // enable component linking