From 9e933efd4b3255c01b5ad08452093c926471d811 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 20 Dec 2024 15:58:05 +0100 Subject: [PATCH] list and describe metadata from oci --- Cargo.lock | 10 +- Cargo.nix | 25 ++- Cargo.toml | 3 +- rust/stackable-cockpit/Cargo.toml | 1 + rust/stackable-cockpit/src/constants.rs | 1 + rust/stackable-cockpit/src/helm.rs | 19 +- rust/stackable-cockpit/src/lib.rs | 1 + rust/stackable-cockpit/src/oci.rs | 173 ++++++++++++++++++ .../src/utils/chartsource.rs | 14 ++ rust/stackable-cockpit/src/utils/mod.rs | 1 + rust/stackablectl/Cargo.toml | 1 + rust/stackablectl/src/cmds/operator.rs | 84 +++++---- 12 files changed, 282 insertions(+), 51 deletions(-) create mode 100644 rust/stackable-cockpit/src/oci.rs create mode 100644 rust/stackable-cockpit/src/utils/chartsource.rs diff --git a/Cargo.lock b/Cargo.lock index e35fbd0a..639975e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -3049,6 +3049,7 @@ dependencies = [ "tokio", "tracing", "url", + "urlencoding", "utoipa", "which", ] @@ -3159,6 +3160,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "urlencoding", ] [[package]] @@ -3762,6 +3764,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" diff --git a/Cargo.nix b/Cargo.nix index c6aa1843..fdaa24a7 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -8052,7 +8052,7 @@ rec { "stream" = [ "tokio/fs" "dep:tokio-util" "dep:wasm-streams" ]; "zstd" = [ "dep:async-compression" "async-compression?/zstd" "dep:tokio-util" ]; }; - resolvedDefaultFeatures = [ "__rustls" "__rustls-ring" "__tls" "blocking" "rustls-tls" "rustls-tls-webpki-roots" ]; + resolvedDefaultFeatures = [ "__rustls" "__rustls-ring" "__tls" "blocking" "json" "rustls-tls" "rustls-tls-webpki-roots" ]; }; "ring" = rec { crateName = "ring"; @@ -9672,7 +9672,7 @@ rec { name = "reqwest"; packageId = "reqwest"; usesDefaultFeatures = false; - features = [ "rustls-tls" ]; + features = [ "json" "rustls-tls" ]; } { name = "semver"; @@ -9722,6 +9722,10 @@ rec { name = "url"; packageId = "url"; } + { + name = "urlencoding"; + packageId = "urlencoding"; + } { name = "utoipa"; packageId = "utoipa"; @@ -10122,7 +10126,7 @@ rec { name = "reqwest"; packageId = "reqwest"; usesDefaultFeatures = false; - features = [ "rustls-tls" ]; + features = [ "json" "rustls-tls" ]; } { name = "semver"; @@ -10177,6 +10181,10 @@ rec { name = "tracing-subscriber"; packageId = "tracing-subscriber"; } + { + name = "urlencoding"; + packageId = "urlencoding"; + } ]; }; @@ -12197,6 +12205,17 @@ rec { }; resolvedDefaultFeatures = [ "default" "serde" ]; }; + "urlencoding" = rec { + crateName = "urlencoding"; + version = "2.1.3"; + edition = "2021"; + sha256 = "1nj99jp37k47n0hvaz5fvz7z6jd0sb4ppvfy3nphr1zbnyixpy6s"; + authors = [ + "Kornel " + "Bertram Truong " + ]; + + }; "utf-8" = rec { crateName = "utf-8"; version = "0.7.6"; diff --git a/Cargo.toml b/Cargo.toml index b0cef9a1..66b79cdb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ phf = "0.11" phf_codegen = "0.11" rand = "0.8" regex = "1.10" -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } rstest = "0.22" semver = { version = "1.0", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } @@ -54,6 +54,7 @@ tower-http = { version = "0.5", features = ["validate-request"] } tracing = "0.1" tracing-subscriber = "0.3" url = "2.5" +urlencoding = "2.1.3" utoipa = { version = "4.2", features = ["indexmap"] } utoipa-swagger-ui = { version = "7.1", features = ["axum"] } uuid = { version = "1.10", features = ["v4"] } diff --git a/rust/stackable-cockpit/Cargo.toml b/rust/stackable-cockpit/Cargo.toml index 6a83a80d..87380fb8 100644 --- a/rust/stackable-cockpit/Cargo.toml +++ b/rust/stackable-cockpit/Cargo.toml @@ -32,6 +32,7 @@ tera.workspace = true tokio.workspace = true tracing.workspace = true url.workspace = true +urlencoding.workspace = true utoipa = { workspace = true, optional = true } which.workspace = true futures.workspace = true diff --git a/rust/stackable-cockpit/src/constants.rs b/rust/stackable-cockpit/src/constants.rs index 82287e18..b0910607 100644 --- a/rust/stackable-cockpit/src/constants.rs +++ b/rust/stackable-cockpit/src/constants.rs @@ -22,6 +22,7 @@ pub const HELM_REPO_NAME_TEST: &str = "stackable-test"; pub const HELM_REPO_NAME_DEV: &str = "stackable-dev"; pub const HELM_REPO_INDEX_FILE: &str = "index.yaml"; +pub const HELM_OCI_BASE: &str = "oci.stackable.tech"; pub const HELM_OCI_REGISTRY: &str = "oci://oci.stackable.tech/sdp-charts"; pub const HELM_DEFAULT_CHART_VERSION: &str = ">0.0.0-0"; diff --git a/rust/stackable-cockpit/src/helm.rs b/rust/stackable-cockpit/src/helm.rs index 58aac1e0..25bca566 100644 --- a/rust/stackable-cockpit/src/helm.rs +++ b/rust/stackable-cockpit/src/helm.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::fmt::Display; use serde::{Deserialize, Serialize}; @@ -7,7 +6,10 @@ use tokio::task::block_in_place; use tracing::{debug, error, info, instrument}; use url::Url; -use crate::constants::{HELM_DEFAULT_CHART_VERSION, HELM_REPO_INDEX_FILE}; +use crate::{ + constants::{HELM_DEFAULT_CHART_VERSION, HELM_REPO_INDEX_FILE}, + utils::chartsource::ChartSourceMetadata, +}; #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -35,17 +37,6 @@ pub struct ChartRepo { pub url: String, } -#[derive(Clone, Debug, Deserialize)] -pub struct Repository { - pub entries: HashMap>, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct RepositoryEntry { - pub name: String, - pub version: String, -} - #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("failed to parse URL"))] @@ -398,7 +389,7 @@ pub fn add_repo(repository_name: &str, repository_url: &str) -> Result<(), Error /// Retrieves the Helm index file from the repository URL. #[instrument] -pub async fn get_helm_index(repo_url: T) -> Result +pub async fn get_helm_index(repo_url: T) -> Result where T: AsRef + std::fmt::Debug, { diff --git a/rust/stackable-cockpit/src/lib.rs b/rust/stackable-cockpit/src/lib.rs index 14220780..e572b560 100644 --- a/rust/stackable-cockpit/src/lib.rs +++ b/rust/stackable-cockpit/src/lib.rs @@ -2,6 +2,7 @@ pub mod common; pub mod constants; pub mod engine; pub mod helm; +pub mod oci; pub mod platform; pub mod utils; pub mod xfer; diff --git a/rust/stackable-cockpit/src/oci.rs b/rust/stackable-cockpit/src/oci.rs new file mode 100644 index 00000000..c6248a5b --- /dev/null +++ b/rust/stackable-cockpit/src/oci.rs @@ -0,0 +1,173 @@ +use std::collections::HashMap; + +use serde::Deserialize; +use snafu::{OptionExt, ResultExt, Snafu}; +use tracing::debug; +use urlencoding::encode; + +use crate::{ + constants::{HELM_OCI_BASE, HELM_REPO_NAME_DEV, HELM_REPO_NAME_STABLE, HELM_REPO_NAME_TEST}, + utils::chartsource::{ChartSourceEntry, ChartSourceMetadata}, +}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("cannot get repositories"))] + GetRepositories { source: reqwest::Error }, + + #[snafu(display("cannot parse repositories"))] + ParseRepositories { source: reqwest::Error }, + + #[snafu(display("cannot get artifacts"))] + GetArtifacts { source: reqwest::Error }, + + #[snafu(display("cannot parse artifacts"))] + ParseArtifacts { source: reqwest::Error }, + + #[snafu(display("unexpected OCI repository name"))] + UnexpectedOciRepositoryName, +} + +#[derive(Deserialize, Debug)] +pub struct OciRepository { + pub name: String, +} + +#[derive(Deserialize, Debug)] +pub struct Tag { + pub name: String, +} + +#[derive(Deserialize, Debug)] +pub struct Artifact { + pub digest: String, + pub tags: Option>, +} + +pub async fn get_oci_index<'a>() -> Result, Error> { + let mut source_index_files: HashMap<&str, ChartSourceMetadata> = HashMap::new(); + + // initialize map + for repo_name in [ + HELM_REPO_NAME_STABLE, + HELM_REPO_NAME_TEST, + HELM_REPO_NAME_DEV, + ] { + source_index_files.insert( + repo_name, + ChartSourceMetadata { + entries: HashMap::new(), + }, + ); + } + let base_url = format!("https://{}/api/v2.0", HELM_OCI_BASE); + + // fetch all operators + let url = format!( + "{}/repositories?page_size={}&q=name=~sdp-charts/", + base_url, 100 + ); + + // reuse connections + let client = reqwest::Client::new(); + + let repositories: Vec = client + .get(&url) + .send() + .await + .context(GetRepositoriesSnafu)? + .json() + .await + .context(ParseRepositoriesSnafu)?; + + debug!("OCI repos {:?}", repositories); + + for repository in &repositories { + // fetch all artifacts pro operator + let (project_name, repository_name) = repository + .name + .split_once('/') + .context(UnexpectedOciRepositoryNameSnafu)?; + + debug!("OCI repo parts {} and {}", project_name, repository_name); + + let mut artifacts = Vec::new(); + let mut page = 1; + let page_size = 20; + + while let Ok(artifacts_page) = client + .get(format!( + "{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}", + base_url, + encode(project_name), + encode(repository_name), + page_size, + page + )) + .send() + .await + .context(GetArtifactsSnafu)? + .json::>() + .await + .context(ParseArtifactsSnafu) + { + let count = artifacts_page.len(); + artifacts.extend(artifacts_page); + if count < page_size { + break; + } + page += 1; + } + + for artifact in &artifacts { + if let Some(release_artifact) = + artifact.tags.as_ref().and_then(|tags| tags.iter().next()) + { + let release_version = release_artifact + .name + .replace("-arm64", "") + .replace("-amd64", ""); + + debug!( + "OCI resolved artifact {}, {}, {}", + release_version.to_string(), + repository_name.to_string(), + release_artifact.name.to_string() + ); + + let entry = ChartSourceEntry { + name: repository_name.to_string(), + version: release_version.to_string(), + }; + + match release_version.as_str() { + "0.0.0-dev" => { + if let Some(repo) = source_index_files.get_mut(HELM_REPO_NAME_DEV) { + repo.entries + .entry(repository_name.to_string()) + .or_default() + .push(entry) + } + } + version if version.contains("-pr") => { + if let Some(repo) = source_index_files.get_mut(HELM_REPO_NAME_TEST) { + repo.entries + .entry(repository_name.to_string()) + .or_default() + .push(entry) + } + } + _ => { + if let Some(repo) = source_index_files.get_mut(HELM_REPO_NAME_STABLE) { + repo.entries + .entry(repository_name.to_string()) + .or_default() + .push(entry) + } + } + } + } + } + } + Ok(source_index_files) +} diff --git a/rust/stackable-cockpit/src/utils/chartsource.rs b/rust/stackable-cockpit/src/utils/chartsource.rs new file mode 100644 index 00000000..5a829e04 --- /dev/null +++ b/rust/stackable-cockpit/src/utils/chartsource.rs @@ -0,0 +1,14 @@ +use std::collections::HashMap; + +use serde::Deserialize; + +#[derive(Clone, Debug, Deserialize)] +pub struct ChartSourceMetadata { + pub entries: HashMap>, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ChartSourceEntry { + pub name: String, + pub version: String, +} diff --git a/rust/stackable-cockpit/src/utils/mod.rs b/rust/stackable-cockpit/src/utils/mod.rs index cad8fa40..a42bcc0d 100644 --- a/rust/stackable-cockpit/src/utils/mod.rs +++ b/rust/stackable-cockpit/src/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod chartsource; pub mod check; pub mod k8s; pub mod params; diff --git a/rust/stackablectl/Cargo.toml b/rust/stackablectl/Cargo.toml index c635ceaa..279d93a5 100644 --- a/rust/stackablectl/Cargo.toml +++ b/rust/stackablectl/Cargo.toml @@ -34,4 +34,5 @@ tracing-subscriber.workspace = true tracing.workspace = true futures.workspace = true termion.workspace = true +urlencoding.workspace = true libc.workspace = true diff --git a/rust/stackablectl/src/cmds/operator.rs b/rust/stackablectl/src/cmds/operator.rs index a848d874..76010d45 100644 --- a/rust/stackablectl/src/cmds/operator.rs +++ b/rust/stackablectl/src/cmds/operator.rs @@ -15,13 +15,15 @@ use stackable_cockpit::{ constants::{ DEFAULT_OPERATOR_NAMESPACE, HELM_REPO_NAME_DEV, HELM_REPO_NAME_STABLE, HELM_REPO_NAME_TEST, }, - helm::{self, Release, Repository}, + helm::{self, Release}, + oci, platform::{ namespace, operator::{self, ChartSourceType}, }, utils::{ self, + chartsource::ChartSourceMetadata, k8s::{self, Client}, }, }; @@ -182,6 +184,9 @@ pub enum CmdError { source: namespace::Error, namespace: String, }, + + #[snafu(display("OCI error"))] + OciError { source: oci::Error }, } /// This list contains a list of operator version grouped by stable, test and @@ -206,12 +211,13 @@ impl OperatorArgs { async fn list_cmd(args: &OperatorListArgs, cli: &Cli) -> Result { debug!("Listing operators"); - // Build map which maps Helm repo name to Helm repo URL - let helm_index_files = build_helm_index_file_list().await?; + // Build map which maps artifacts to a chart source + let source_index_files = + build_source_index_file_list(&ChartSourceType::from(args.chart_source.clone())).await?; // Iterate over all valid operators and create a list of versions grouped // by stable, test and dev lines - let versions_list = build_versions_list(&helm_index_files)?; + let versions_list = build_versions_list(&source_index_files)?; match args.output_type { OutputType::Plain | OutputType::Table => { @@ -262,11 +268,12 @@ async fn list_cmd(args: &OperatorListArgs, cli: &Cli) -> Result Result { debug!("Describing operator {}", args.operator_name); - // Build map which maps Helm repo name to Helm repo URL - let helm_index_files = build_helm_index_file_list().await?; + // Build map which maps artifacts to a chart source + let source_index_files = + build_source_index_file_list(&ChartSourceType::from(args.chart_source.clone())).await?; // Create a list of versions for this operator - let versions_list = build_versions_list_for_operator(&args.operator_name, &helm_index_files)?; + let versions_list = build_versions_list_for_operator(&args.operator_name, &source_index_files)?; match args.output_type { OutputType::Plain | OutputType::Table => { @@ -465,37 +472,50 @@ fn installed_cmd(args: &OperatorInstalledArgs, cli: &Cli) -> Result() -> Result, CmdError> { - debug!("Building Helm index file list"); - - let mut helm_index_files = HashMap::new(); - - for helm_repo_name in [ - HELM_REPO_NAME_STABLE, - HELM_REPO_NAME_TEST, - HELM_REPO_NAME_DEV, - ] { - let helm_repo_url = - helm_repo_name_to_repo_url(helm_repo_name).context(InvalidRepoNameSnafu)?; - - helm_index_files.insert( - helm_repo_name, - helm::get_helm_index(helm_repo_url) - .await - .context(HelmSnafu)?, - ); - } +async fn build_source_index_file_list<'a>( + chart_source: &ChartSourceType, +) -> Result, CmdError> { + debug!("Building source index file list"); + + let mut source_index_files: HashMap<&str, ChartSourceMetadata> = HashMap::new(); + + match chart_source { + ChartSourceType::OCI => { + source_index_files = oci::get_oci_index().await.context(OciSnafu)?; + + debug!("OCI Repository entries: {:?}", source_index_files); + } + ChartSourceType::Repo => { + for helm_repo_name in [ + HELM_REPO_NAME_STABLE, + HELM_REPO_NAME_TEST, + HELM_REPO_NAME_DEV, + ] { + let helm_repo_url = + helm_repo_name_to_repo_url(helm_repo_name).context(InvalidRepoNameSnafu)?; + + source_index_files.insert( + helm_repo_name, + helm::get_helm_index(helm_repo_url) + .await + .context(HelmSnafu)?, + ); + + debug!("Helm Repository entries: {:?}", source_index_files); + } + } + }; - Ok(helm_index_files) + Ok(source_index_files) } /// Iterates over all valid operators and creates a list of versions grouped /// by stable, test and dev lines based on the list of Helm repo index files. #[instrument] fn build_versions_list( - helm_index_files: &HashMap<&str, Repository>, + helm_index_files: &HashMap<&str, ChartSourceMetadata>, ) -> Result, CmdError> { debug!("Building versions list"); @@ -518,7 +538,7 @@ fn build_versions_list( #[instrument] fn build_versions_list_for_operator( operator_name: T, - helm_index_files: &HashMap<&str, Repository>, + helm_index_files: &HashMap<&str, ChartSourceMetadata>, ) -> Result where T: AsRef + std::fmt::Debug, @@ -543,7 +563,7 @@ where #[instrument] fn list_operator_versions_from_repo( operator_name: T, - helm_repo: &Repository, + helm_repo: &ChartSourceMetadata, ) -> Result, CmdError> where T: AsRef + std::fmt::Debug,