Merge pull request #387 from cbgbt/bundle-krane

Bump the Kit metadata label version, bundle krane with Twoliter

.gitignore

@@ -1,3 +1,6 @@
 /target/
 .ignore/
-twoliter-attributions.tar.gz
+*.tar.gz
+/tools/krane/go-containerregistry-*
+/tools/krane/krane
+/tools/krane/krane.gz

Cargo.toml

@@ -41,6 +41,7 @@ bottlerocket-types = { version = "0.0.14", git = "https://github.com/bottlerocke
 bottlerocket-variant = { version = "0.1", path = "tools/bottlerocket-variant" }
 buildsys = { version = "0.1", path = "tools/buildsys", lib = true, artifact = [ "bin:buildsys" ] }
 buildsys-config = { version = "0.1", path = "tools/buildsys-config" }
+krane-bundle = { version = "0.1", path = "tools/krane" }
 oci-cli-wrapper = { version = "0.1", path = "tools/oci-cli-wrapper" }
 parse-datetime = { version = "0.1", path = "tools/parse-datetime" }
 pipesys = { version = "0.1", path = "tools/pipesys", lib = true, artifact = [ "bin:pipesys" ] }
@@ -88,15 +89,18 @@ home = "0.5"
 indicatif = "0.17"
 inotify = "0.10.2"
 lazy_static = "1"
+libc = "0.2"
 log = "0.4"
 maplit = "1"
 nix = "0.28"
 nonzero_ext = "0.3"
 num_cpus = "1"
 olpc-cjson = "0.1"
+pentacle = "1.1"
 rand = { version = "0.8", default-features = false }
 regex = "1"
 reqwest = { version = "0.11", default-features = false }
+seccompiler = "0.4"
 semver = "1"
 serde = "1"
 serde_json = "1"

tests/integration-tests/Cargo.toml

@@ -5,5 +5,8 @@ edition = "2021"
 license = "MIT OR Apache-2.0"
 
 [dev-dependencies]
+libc.workspace = true
+tempfile.workspace = true
 tokio = { workspace = true, features = ["fs", "process", "rt-multi-thread"] }
+toml.workspace = true
 twoliter = { workspace = true }

tests/integration-tests/src/lib.rs

@@ -2,7 +2,8 @@
 
 use std::ffi::OsStr;
 use std::path::PathBuf;
-use tokio::process::Command;
+use std::process::Command;
+use tempfile::TempDir;
 
 mod twoliter_update;
 
@@ -14,7 +15,7 @@ pub fn test_projects_dir() -> PathBuf {
     p.join("projects")
 }
 
-pub async fn run_command<I, S, E>(cmd: S, args: I, env: E) -> std::process::Output
+pub fn run_command<I, S, E>(cmd: S, args: I, env: E) -> std::process::Output
 where
     I: IntoIterator<Item = S>,
     E: IntoIterator<Item = (S, S)>,
@@ -35,10 +36,88 @@ where
         .args(args.into_iter())
         .envs(env.into_iter())
         .output()
-        .await
         .expect("failed to execute process");
 
     println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
     println!("stderr: {}", String::from_utf8_lossy(&output.stderr));
     output
 }
+
+struct KitRegistry {
+    temp_dir: TempDir,
+    container_id: String,
+}
+
+impl KitRegistry {
+    fn new() -> Self {
+        let temp_dir = TempDir::new().expect("failed to create path for oci registry spinup");
+
+        let cert_dir = temp_dir.path().join("certs");
+        let cert_file = cert_dir.join("registry.crt");
+        std::fs::create_dir_all(&cert_dir).expect("failed to create nginx dir");
+        let output = run_command(
+            "openssl",
+            [
+                "req",
+                "-x509",
+                "-nodes",
+                "-days",
+                "365",
+                "-newkey",
+                "rsa:2048",
+                "-keyout",
+                cert_dir.join("registry.key").to_str().unwrap(),
+                "-out",
+                cert_file.to_str().unwrap(),
+                "-batch",
+                "-addext",
+                "subjectAltName=DNS:localhost",
+            ],
+            [],
+        );
+        assert!(
+            output.status.success(),
+            "generate openssl self-signed certificates"
+        );
+
+        let output = run_command(
+            "docker",
+            [
+                "run",
+                "-d",
+                "--rm",
+                "--volume",
+                "./certs:/auth/certs",
+                "-e REGISTRY_HTTP_RELATIVEURLS=true",
+                "-e REGISTRY_HTTP_ADDR=0.0.0.0:5000",
+                "-e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/registry.crt",
+                "-e REGISTRY_HTTP_TLS_KEY=/auth/certs/registry.key",
+                "-p",
+                "5000:5000",
+                "public.ecr.aws/docker/library/registry:2.8.3",
+            ],
+            [],
+        );
+        assert!(output.status.success(), "failed to start oci registry");
+        let container_id = String::from_utf8(output.stdout).unwrap().trim().to_string();
+
+        Self {
+            temp_dir,
+            container_id,
+        }
+    }
+
+    fn cert_file(&self) -> PathBuf {
+        self.temp_dir
+            .path()
+            .join("certs/registry.crt")
+            .to_path_buf()
+    }
+}
+
+impl Drop for KitRegistry {
+    fn drop(&mut self) {
+        let output = run_command("docker", ["kill", &self.container_id], []);
+        assert!(output.status.success(), "failed to stop oci registry");
+    }
+}

tests/integration-tests/src/twoliter_update.rs

@@ -1,74 +1,134 @@
-use super::{run_command, test_projects_dir, TWOLITER_PATH};
-
-const EXPECTED_LOCKFILE: &str = r#"schema-version = 1
-
-[sdk]
-name = "bottlerocket-sdk"
-version = "0.42.0"
-vendor = "bottlerocket"
-source = "public.ecr.aws/bottlerocket/bottlerocket-sdk:v0.42.0"
-digest = "myHHKE41h9qfeyR6V6HB0BfiLPwj3QEFLUFy4TXcR10="
-
-[[kit]]
-name = "bottlerocket-core-kit"
-version = "2.0.0"
-vendor = "custom-vendor"
-source = "public.ecr.aws/bottlerocket/bottlerocket-core-kit:v2.0.0"
-digest = "vlTsAAbSCzXFZofVmw8pLLkRjnG/y8mtb2QsQBSz1zk="
+use super::{run_command, test_projects_dir, KitRegistry, TWOLITER_PATH};
+
+const INFRA_TOML: &str = r#"
+[vendor.bottlerocket]
+registry = "localhost:5000"
+"#;
+
+const TWOLITER_OVERRIDE: &str = r#"
+[custom-vendor.core-kit]
+registry = "localhost:5000"
+name = "core-kit-overridden"
 "#;
 
-#[tokio::test]
+#[test]
 #[ignore]
-/// Generates a Twoliter.lock file for the `external-kit` project using docker
-async fn test_twoliter_update_docker() {
+/// Generates a Twoliter.lock file for the `external-kit` project using crane
+fn test_twoliter_build_and_update() {
     let external_kit = test_projects_dir().join("external-kit");
-
     let lockfile = external_kit.join("Twoliter.lock");
-    tokio::fs::remove_file(&lockfile).await.ok();
+    std::fs::remove_file(&lockfile).ok();
+    let override_file = external_kit.join("Twoliter.override");
+    std::fs::remove_file(&override_file).ok();
+
+    // Build & push a local kit to the registry
+    let registry = KitRegistry::new();
+    LocalKit::build(&registry);
 
+    // Point twoliter to the local registry as an override
+    std::fs::write(&override_file, TWOLITER_OVERRIDE).unwrap();
     let output = run_command(
         TWOLITER_PATH,
         [
             "update",
             "--project-path",
             external_kit.join("Twoliter.toml").to_str().unwrap(),
         ],
-        [("TWOLITER_KIT_IMAGE_TOOL", "docker")],
-    )
-    .await;
+        [
+            ("TWOLITER_KIT_IMAGE_TOOL", "crane"),
+            ("SSL_CERT_FILE", registry.cert_file().to_str().unwrap()),
+        ],
+    );
 
     assert!(output.status.success());
 
-    let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap();
-    assert_eq!(lock_contents, EXPECTED_LOCKFILE);
+    // Assert that we successfully create a lock
+    let lock_contents = std::fs::read_to_string(&lockfile).unwrap();
+    let parsed: toml::Value = toml::from_str(&lock_contents).unwrap();
+    let kits = parsed
+        .as_table()
+        .unwrap()
+        .get("kit")
+        .unwrap()
+        .as_array()
+        .unwrap();
+
+    assert_eq!(kits.len(), 1);
+    let core_kit = kits[0].as_table().unwrap();
+    assert_eq!(core_kit.get("name").unwrap().as_str().unwrap(), "core-kit");
+    assert_eq!(core_kit.get("version").unwrap().as_str().unwrap(), "1.0.0");
+    assert_eq!(
+        core_kit.get("vendor").unwrap().as_str().unwrap(),
+        "custom-vendor"
+    );
+    assert_eq!(
+        core_kit.get("source").unwrap().as_str().unwrap(),
+        "definitely-wont-resolve/core-kit:v1.0.0"
+    );
 
-    tokio::fs::remove_file(&lockfile).await.ok();
+    std::fs::remove_file(&lockfile).ok();
+    std::fs::remove_file(&override_file).ok();
 }
 
-#[tokio::test]
-#[ignore]
-/// Generates a Twoliter.lock file for the `external-kit` project using crane
-async fn test_twoliter_update_crane() {
-    let external_kit = test_projects_dir().join("external-kit");
+struct LocalKit;
 
-    let lockfile = external_kit.join("Twoliter.lock");
-    tokio::fs::remove_file(&lockfile).await.ok();
+impl LocalKit {
+    fn build(registry: &KitRegistry) {
+        let local_kit = test_projects_dir().join("local-kit");
 
-    let output = run_command(
-        TWOLITER_PATH,
-        [
-            "update",
-            "--project-path",
-            external_kit.join("Twoliter.toml").to_str().unwrap(),
-        ],
-        [("TWOLITER_KIT_IMAGE_TOOL", "crane")],
-    )
-    .await;
+        run_command(
+            TWOLITER_PATH,
+            [
+                "update",
+                "--project-path",
+                local_kit.join("Twoliter.toml").to_str().unwrap(),
+            ],
+            [],
+        );
 
-    assert!(output.status.success());
+        run_command(
+            TWOLITER_PATH,
+            [
+                "fetch",
+                "--project-path",
+                local_kit.join("Twoliter.toml").to_str().unwrap(),
+            ],
+            [],
+        );
 
-    let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap();
-    assert_eq!(lock_contents, EXPECTED_LOCKFILE);
+        run_command(
+            TWOLITER_PATH,
+            [
+                "build",
+                "kit",
+                "core-kit",
+                "--project-path",
+                local_kit.join("Twoliter.toml").to_str().unwrap(),
+            ],
+            [],
+        );
+
+        std::fs::write(local_kit.join("Infra.toml"), INFRA_TOML).unwrap();
+        run_command(
+            TWOLITER_PATH,
+            [
+                "publish",
+                "kit",
+                "--project-path",
+                local_kit.join("Twoliter.toml").to_str().unwrap(),
+                "core-kit",
+                "bottlerocket",
+                "core-kit-overridden",
+            ],
+            [("SSL_CERT_FILE", registry.cert_file().to_str().unwrap())],
+        );
+    }
+}
 
-    tokio::fs::remove_file(&lockfile).await.ok();
+impl Drop for LocalKit {
+    fn drop(&mut self) {
+        let local_kit = test_projects_dir().join("local-kit");
+        std::fs::remove_file(local_kit.join("Twoliter.lock")).ok();
+        std::fs::remove_file(local_kit.join("Infra.toml")).ok();
+    }
 }

tests/projects/external-kit/.gitignore

@@ -8,3 +8,5 @@
 Test.toml
 testsys.kubeconfig
 Infra.toml
+Twoliter.lock
+Twoliter.override