From 1fc24d445461284413d0135f34c8e9768a48d197 Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Sun, 17 Nov 2024 22:08:46 +0000 Subject: [PATCH 01/12] datastore: changes to match changes in Core kit datastore Updated the `commit_transaction` function to enable committing metadata from pending transactions. In commit transaction we will first commit metadata and then pending keys to correctly perform the check to identify if key exists or not. The strength handling among pending and committed transaction is as: If pending metadata is strong and committed metadata is weak, commit the pending setting. If pending metadata is weak, committed metadata is strong and the setting is already available, do not downgrade from strong to weak. Otherwise add the strength file with value as weak. If pending and committed metadata are the same, no action is performed. Additionally, made minor changes to metadata functions for improved access and flexibility: Introduced a `committed` field to dynamically access metadata in pending transactions. Replaced the hardcoded use of live committed metadata with this committed variable ans pass Committed::Live from previous usages. Refer commit: https://github.com/bottlerocket-os/bottlerocket-core-kit/pull/294/commits/20a435e3f06625f6c75ccf8109a4a722a055d229 Refer PR: https://github.com/bottlerocket-os/bottlerocket-core-kit/pull/294 --- sources/api/datastore/src/error.rs | 6 ++ sources/api/datastore/src/filesystem.rs | 108 ++++++++++++++++++++++-- sources/api/datastore/src/lib.rs | 53 ++++++++---- sources/api/datastore/src/memory.rs | 89 +++++++++++++++---- 4 files changed, 219 insertions(+), 37 deletions(-) diff --git a/sources/api/datastore/src/error.rs b/sources/api/datastore/src/error.rs index d1c7207ae1a..86d386b52ad 100644 --- a/sources/api/datastore/src/error.rs +++ b/sources/api/datastore/src/error.rs @@ -59,6 +59,12 @@ pub enum Error { #[snafu(display("Key name beyond maximum length {}: {}", name, max))] KeyTooLong { name: String, max: usize }, + + #[snafu(display("Unable to serialize data: {}", source))] + Serialize { source: serde_json::Error }, + + #[snafu(display("Unable to de-serialize data: {}", source))] + DeSerialize { source: serde_json::Error }, } pub type Result = std::result::Result; diff --git a/sources/api/datastore/src/filesystem.rs b/sources/api/datastore/src/filesystem.rs index bf89eed783f..99bba948e5c 100644 --- a/sources/api/datastore/src/filesystem.rs +++ b/sources/api/datastore/src/filesystem.rs @@ -4,7 +4,7 @@ //! Data is kept in files with paths resembling the keys, e.g. a/b/c for a.b.c, and metadata is //! kept in a suffixed file next to the data, e.g. a/b/c.meta for metadata "meta" about a.b.c -use log::{debug, error, trace}; +use log::{debug, error, trace, warn}; use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC}; use snafu::{ensure, OptionExt, ResultExt}; use std::collections::{HashMap, HashSet}; @@ -13,6 +13,8 @@ use std::io; use std::path::{self, Path, PathBuf}; use walkdir::{DirEntry, WalkDir}; +use crate::{deserialize_scalar, serialize_scalar, ScalarError}; + use super::key::{Key, KeyType}; use super::{error, Committed, DataStore, Result}; @@ -413,6 +415,7 @@ impl DataStore for FilesystemDataStore { fn list_populated_metadata( &self, prefix: S1, + committed: &Committed, metadata_key_name: &Option, ) -> Result>> where @@ -420,7 +423,7 @@ impl DataStore for FilesystemDataStore { S2: AsRef, { // Find metadata key paths on disk - let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, &Committed::Live)?; + let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, committed)?; // For each file on disk, check the user's conditions, and add it to our output let mut result = HashMap::new(); @@ -460,8 +463,13 @@ impl DataStore for FilesystemDataStore { self.delete_key_path(path, committed) } - fn get_metadata_raw(&self, metadata_key: &Key, data_key: &Key) -> Result> { - let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?; + fn get_metadata_raw( + &self, + metadata_key: &Key, + data_key: &Key, + committed: &Committed, + ) -> Result> { + let path = self.metadata_path(metadata_key, data_key, committed)?; read_file_for_key(metadata_key, &path) } @@ -470,8 +478,9 @@ impl DataStore for FilesystemDataStore { metadata_key: &Key, data_key: &Key, value: S, + committed: &Committed, ) -> Result<()> { - let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?; + let path = self.metadata_path(metadata_key, data_key, committed)?; write_file_mkdir(path, value) } @@ -489,6 +498,95 @@ impl DataStore for FilesystemDataStore { let pending = Committed::Pending { tx: transaction.into(), }; + + // We will first commit metadata to correctly check that if a setting exist and + // its strength is strong, we will not change it to weak. + // Get metadata from pending transaction + let transaction_metadata = + self.get_metadata_prefix("settings.", &pending, &None as &Option<&str>)?; + + trace!( + "commit_transaction: transaction_metadata: {:?}", + transaction_metadata + ); + + for (key, value) in transaction_metadata { + for (metadata_key, metadata_value) in value { + // For now we are only processing the strength metadata from pending + // transaction to live + if metadata_key.name() != "strength" { + continue; + } + + // strength in pending transaction + let pending_strength: String = + deserialize_scalar::<_, ScalarError>(&metadata_value.clone()) + .with_context(|_| error::DeSerializeSnafu {})?; + + // Get the setting strength in live + // get_metadata function returns Ok(None) in case strength does not exist + // We will consider this case as strength equals strong. + let committed_strength = + match self.get_metadata(&metadata_key, &key, &Committed::Live) { + Ok(Some(v)) => v, + Ok(None) => "strong".to_string(), + Err(_) => continue, + }; + + // The get key funtion returns Ok(None) in case if the path does not exist + // and error if some path exist and some error occurred in fetching + // Hence we we will return error in case of error + // from get key function and continue to add/change to weak key + // if the value is None. + let value = self.get_key(&key, &Committed::Live)?; + + trace!( + "commit_transaction: key: {:?}, metadata_key: {:?}, metadata_value: {:?}", + key.name(), + metadata_key.name(), + metadata_value + ); + + match (pending_strength.as_str(), committed_strength.as_str()) { + ("weak", "strong") => { + // Do not change from strong to weak if setting exists + // otherwise commit strength metadata with value as "weak" + if value.is_some() { + warn!("Trying to change the strength from strong to weak for key: {}, Operation ignored", key.name()); + continue; + } else { + let met_value = serialize_scalar::<_, ScalarError>(&pending_strength) + .with_context(|_| error::SerializeSnafu {})?; + + self.set_metadata(&metadata_key, &key, met_value, &Committed::Live)?; + } + } + ("strong", "weak") => { + let met_value = serialize_scalar::<_, ScalarError>(&pending_strength) + .with_context(|_| error::SerializeSnafu {})?; + self.set_metadata(&metadata_key, &key, met_value, &Committed::Live)?; + } + ("weak", "weak") => { + trace!("The strength for setting {} is already weak", key.name()); + continue; + } + ("strong", "strong") => { + trace!("The strength for setting {} is already strong", key.name()); + continue; + } + _ => { + warn!( + "The strength for setting {} is not one of weak or strong. Pending strength: {}, Committed strength: {} ", + key.name(), + pending_strength, + committed_strength + ); + continue; + } + }; + } + } + // Get data for changed keys let pending_data = self.get_prefix("settings.", &pending)?; diff --git a/sources/api/datastore/src/lib.rs b/sources/api/datastore/src/lib.rs index 9156a525a19..3d375d35cf4 100644 --- a/sources/api/datastore/src/lib.rs +++ b/sources/api/datastore/src/lib.rs @@ -72,6 +72,7 @@ pub trait DataStore { fn list_populated_metadata( &self, prefix: S1, + committed: &Committed, metadata_key_name: &Option, ) -> Result>> where @@ -89,7 +90,12 @@ pub trait DataStore { /// Retrieve the value for a single metadata key from the datastore. Values will inherit from /// earlier in the tree, if more specific values are not found later. - fn get_metadata(&self, metadata_key: &Key, data_key: &Key) -> Result> { + fn get_metadata( + &self, + metadata_key: &Key, + data_key: &Key, + committed: &Committed, + ) -> Result> { let mut result = Ok(None); let mut current_path = Vec::new(); @@ -101,7 +107,7 @@ pub trait DataStore { unreachable!("Prefix of Key failed to make Key: {:?}", current_path) }); - if let Some(md) = self.get_metadata_raw(metadata_key, &data_key)? { + if let Some(md) = self.get_metadata_raw(metadata_key, &data_key, committed)? { result = Ok(Some(md)); } } @@ -110,13 +116,19 @@ pub trait DataStore { /// Retrieve the value for a single metadata key from the datastore, without taking into /// account inheritance of metadata from earlier in the tree. - fn get_metadata_raw(&self, metadata_key: &Key, data_key: &Key) -> Result>; + fn get_metadata_raw( + &self, + metadata_key: &Key, + data_key: &Key, + committed: &Committed, + ) -> Result>; /// Set the value of a single metadata key in the datastore. fn set_metadata>( &mut self, metadata_key: &Key, data_key: &Key, value: S, + committed: &Committed, ) -> Result<()>; /// Removes the given metadata key from the given data key in the datastore. If we /// succeeded, we return Ok(()); if the data or metadata key didn't exist, we also return @@ -205,13 +217,14 @@ pub trait DataStore { fn get_metadata_prefix( &self, find_prefix: S1, + committed: &Committed, metadata_key_name: &Option, ) -> Result>> where S1: AsRef, S2: AsRef, { - let meta_map = self.list_populated_metadata(&find_prefix, metadata_key_name)?; + let meta_map = self.list_populated_metadata(&find_prefix, committed, metadata_key_name)?; trace!("Found populated metadata: {:?}", meta_map); if meta_map.is_empty() { return Ok(HashMap::new()); @@ -234,12 +247,12 @@ pub trait DataStore { meta_key, &data_key ); - let value = self.get_metadata(&meta_key, &data_key)?.context( - error::ListedMetaNotPresentSnafu { + let value = self + .get_metadata(&meta_key, &data_key, committed)? + .context(error::ListedMetaNotPresentSnafu { meta_key: meta_key.name(), data_key: data_key.name(), - }, - )?; + })?; // Insert a top-level map entry for the data key if we've found metadata. let data_entry = result.entry(data_key.clone()).or_insert_with(HashMap::new); @@ -336,14 +349,20 @@ mod test { let grandchild = Key::new(KeyType::Data, "a.b.c").unwrap(); // Set metadata on parent - m.set_metadata(&meta, &parent, "value").unwrap(); + m.set_metadata(&meta, &parent, "value", &Committed::Live) + .unwrap(); // Metadata shows up on grandchild... assert_eq!( - m.get_metadata(&meta, &grandchild).unwrap(), + m.get_metadata(&meta, &grandchild, &Committed::Live) + .unwrap(), Some("value".to_string()) ); // ...but only through inheritance, not directly. - assert_eq!(m.get_metadata_raw(&meta, &grandchild).unwrap(), None); + assert_eq!( + m.get_metadata_raw(&meta, &grandchild, &Committed::Live) + .unwrap(), + None + ); } #[test] @@ -379,20 +398,22 @@ mod test { let mk1 = Key::new(KeyType::Meta, "metatest1").unwrap(); let mk2 = Key::new(KeyType::Meta, "metatest2").unwrap(); let mk3 = Key::new(KeyType::Meta, "metatest3").unwrap(); - m.set_metadata(&mk1, &k1, "41").unwrap(); - m.set_metadata(&mk2, &k2, "42").unwrap(); - m.set_metadata(&mk3, &k3, "43").unwrap(); + m.set_metadata(&mk1, &k1, "41", &Committed::Live).unwrap(); + m.set_metadata(&mk2, &k2, "42", &Committed::Live).unwrap(); + m.set_metadata(&mk3, &k3, "43", &Committed::Live).unwrap(); // Check all metadata assert_eq!( - m.get_metadata_prefix("x.", &None as &Option<&str>).unwrap(), + m.get_metadata_prefix("x.", &Committed::Live, &None as &Option<&str>) + .unwrap(), hashmap!(k1 => hashmap!(mk1 => "41".to_string()), k2.clone() => hashmap!(mk2.clone() => "42".to_string())) ); // Check metadata matching a given name assert_eq!( - m.get_metadata_prefix("x.", &Some("metatest2")).unwrap(), + m.get_metadata_prefix("x.", &Committed::Live, &Some("metatest2")) + .unwrap(), hashmap!(k2 => hashmap!(mk2 => "42".to_string())) ); } diff --git a/sources/api/datastore/src/memory.rs b/sources/api/datastore/src/memory.rs index 4ffc397912f..812272d85eb 100644 --- a/sources/api/datastore/src/memory.rs +++ b/sources/api/datastore/src/memory.rs @@ -16,6 +16,9 @@ pub struct MemoryDataStore { // Map of data keys to their metadata, which in turn is a mapping of metadata keys to // arbitrary (string/serialized) values. metadata: HashMap>, + // Map of data keys to their metadata, which in turn is a mapping of metadata keys to + // arbitrary (string/serialized) values in pending transaction + pending_metadata: HashMap>, } impl MemoryDataStore { @@ -57,14 +60,20 @@ impl DataStore for MemoryDataStore { fn list_populated_metadata( &self, prefix: S1, + committed: &Committed, metadata_key_name: &Option, ) -> Result>> where S1: AsRef, S2: AsRef, { + let metadata_to_use = match committed { + Committed::Live => &self.metadata, + Committed::Pending { .. } => &self.pending_metadata, + }; + let mut result = HashMap::new(); - for (data_key, meta_map) in self.metadata.iter() { + for (data_key, meta_map) in metadata_to_use.iter() { // Confirm data key matches requested prefix. if !data_key.name().starts_with(prefix.as_ref()) { continue; @@ -112,8 +121,18 @@ impl DataStore for MemoryDataStore { Ok(dataset.contains_key(key)) } - fn get_metadata_raw(&self, metadata_key: &Key, data_key: &Key) -> Result> { - let metadata_for_data = self.metadata.get(data_key); + fn get_metadata_raw( + &self, + metadata_key: &Key, + data_key: &Key, + committed: &Committed, + ) -> Result> { + let metadata_to_use = match committed { + Committed::Live => &self.metadata, + Committed::Pending { .. } => &self.pending_metadata, + }; + + let metadata_for_data = metadata_to_use.get(data_key); // If we have a metadata entry for this data key, then we can try fetching the requested // metadata key, otherwise we'll return early with Ok(None). let result = metadata_for_data.and_then(|m| m.get(metadata_key)); @@ -125,17 +144,14 @@ impl DataStore for MemoryDataStore { metadata_key: &Key, data_key: &Key, value: S, + committed: &Committed, ) -> Result<()> { - // If we don't already have a metadata entry for this data key, insert one. - let metadata_for_data = self - .metadata - // Clone data key because we want the HashMap key type to be Key, not &Key, and we - // can't pass ownership because we only have a reference from our parameters. - .entry(data_key.clone()) - .or_default(); - - metadata_for_data.insert(metadata_key.clone(), value.as_ref().to_owned()); - Ok(()) + match committed { + Committed::Live => set_metadata_raw(&mut self.metadata, metadata_key, data_key, value), + Committed::Pending { .. } => { + set_metadata_raw(&mut self.pending_metadata, metadata_key, data_key, value) + } + } } fn unset_metadata(&mut self, metadata_key: &Key, data_key: &Key) -> Result<()> { @@ -179,6 +195,23 @@ impl DataStore for MemoryDataStore { } } +fn set_metadata_raw>( + metadata_to_use: &mut HashMap>, + metadata_key: &Key, + data_key: &Key, + value: S, +) -> Result<()> { + // If we don't already have a metadata entry for this data key, insert one. + let metadata_for_data = metadata_to_use + // Clone data key because we want the HashMap key type to be Key, not &Key, and we + // can't pass ownership because we only have a reference from our parameters. + .entry(data_key.clone()) + .or_default(); + + metadata_for_data.insert(metadata_key.clone(), value.as_ref().to_owned()); + Ok(()) +} + #[cfg(test)] mod test { use super::super::{Committed, DataStore, Key, KeyType}; @@ -198,14 +231,38 @@ mod test { let mdkey = Key::new(KeyType::Meta, "testmd").unwrap(); let md = "mdval"; - m.set_metadata(&mdkey, &k, md).unwrap(); + m.set_metadata(&mdkey, &k, md, &Committed::Live).unwrap(); assert_eq!( - m.get_metadata_raw(&mdkey, &k).unwrap(), + m.get_metadata_raw(&mdkey, &k, &Committed::Live).unwrap(), + Some(md.to_string()) + ); + + m.set_metadata( + &mdkey, + &k, + md, + &Committed::Pending { + tx: "test".to_owned(), + }, + ) + .unwrap(); + assert_eq!( + m.get_metadata_raw( + &mdkey, + &k, + &Committed::Pending { + tx: "test".to_owned() + } + ) + .unwrap(), Some(md.to_string()) ); m.unset_metadata(&mdkey, &k).unwrap(); - assert_eq!(m.get_metadata_raw(&mdkey, &k).unwrap(), None); + assert_eq!( + m.get_metadata_raw(&mdkey, &k, &Committed::Live).unwrap(), + None + ); m.unset_key(&k, &Committed::Live).unwrap(); assert_eq!(m.get_key(&k, &Committed::Live).unwrap(), None); From 181bb585ff5a0cdde9ab29fbfa8fcbefdd064cf6 Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Sun, 17 Nov 2024 23:04:45 +0000 Subject: [PATCH 02/12] shared-defaults: change aws host container source setting-generator to table Update the source from string like: schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-admin:v0.11.14' To setting generator as object, that contains: - command - strength - skip-if-populated --- sources/shared-defaults/aws-host-containers.toml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sources/shared-defaults/aws-host-containers.toml b/sources/shared-defaults/aws-host-containers.toml index 7d11572af7b..38d096edb2f 100644 --- a/sources/shared-defaults/aws-host-containers.toml +++ b/sources/shared-defaults/aws-host-containers.toml @@ -2,8 +2,10 @@ enabled = false superpowered = true -[metadata.settings.host-containers.admin.source] -setting-generator = "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-admin:v0.11.14'" +[metadata.settings.host-containers.admin.source.setting-generator] +command = "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-admin:v0.11.14'" +strength = "weak" +skip-if-populated = true [metadata.settings.host-containers.admin.user-data] setting-generator = "shibaken generate-admin-userdata" @@ -12,5 +14,8 @@ setting-generator = "shibaken generate-admin-userdata" enabled = true superpowered = false -[metadata.settings.host-containers.control.source] -setting-generator = "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-control:v0.7.18'" +[metadata.settings.host-containers.control.source.setting-generator] +command = "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-control:v0.7.18'" +strength = "weak" +skip-if-populated = true + From b9fbb93755d7a76fbf7ef56597a2c9f60494b2fd Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Wed, 20 Nov 2024 19:00:09 +0000 Subject: [PATCH 03/12] shared-defaults: change public host container source setting-generator to table Update the source from string like: public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.14 To setting generator as object, that contains: - command - strength - skip-if-populated --- sources/shared-defaults/public-host-containers.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sources/shared-defaults/public-host-containers.toml b/sources/shared-defaults/public-host-containers.toml index 5ba49a64d4b..bcbb484a341 100644 --- a/sources/shared-defaults/public-host-containers.toml +++ b/sources/shared-defaults/public-host-containers.toml @@ -6,9 +6,17 @@ [settings.host-containers.admin] enabled = false superpowered = true -source = "public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.14" + +[metadata.settings.host-containers.admin.source.setting-generator] +command = "schnauzer-v2 render --template 'public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.14'" +strength = "weak" +skip-if-populated = true [settings.host-containers.control] enabled = false superpowered = false -source = "public.ecr.aws/bottlerocket/bottlerocket-control:v0.7.18" + +[metadata.settings.host-containers.control.source.setting-generator] +command = "schnauzer-v2 render --template 'public.ecr.aws/bottlerocket/bottlerocket-control:v0.7.18'" +strength = "weak" +skip-if-populated = true From 5aa44d36ca83164bfbfd27a0bd1010c3e8bc3825 Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Thu, 5 Dec 2024 20:08:05 +0000 Subject: [PATCH 04/12] apiclient: update README to document version 2 of /tx API --- sources/api/apiclient/README.md | 4 ++++ sources/api/apiclient/README.tpl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/sources/api/apiclient/README.md b/sources/api/apiclient/README.md index dea3dfc89e6..78694f7bbdf 100644 --- a/sources/api/apiclient/README.md +++ b/sources/api/apiclient/README.md @@ -174,6 +174,10 @@ You can see all your pending settings like this: ```shell apiclient raw -u /tx ``` +You can also see pending metadata along with pending setting using version 2 of `/tx` like this: +```shell +apiclient raw -u /v2/tx +``` To *commit* the settings, and let the system apply them to any relevant configuration files or services, do this: ```shell diff --git a/sources/api/apiclient/README.tpl b/sources/api/apiclient/README.tpl index 9604d7f6869..e886246a40c 100644 --- a/sources/api/apiclient/README.tpl +++ b/sources/api/apiclient/README.tpl @@ -174,6 +174,10 @@ You can see all your pending settings like this: ```shell apiclient raw -u /tx ``` +You can also see pending metadata along with pending setting using version 2 of `/tx` like this: +```shell +apiclient raw -u /v2/tx +``` To *commit* the settings, and let the system apply them to any relevant configuration files or services, do this: ```shell From 52fa106b18e65e565caaa66f821b5c839bb5142a Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 22 Nov 2024 18:29:28 +0000 Subject: [PATCH 05/12] models: Add Strength enum and Setting-Generator struct We need to add these to match changes in Bottlerocket-core-kit Refer commit: https://github.com/bottlerocket-os/bottlerocket-core-kit/pull/294/commits/a72f6bd16f7de0d628e80fb03bbf827ebd4892e2 PR: https://github.com/bottlerocket-os/bottlerocket-core-kit/pull/294 --- sources/Cargo.lock | 1 + sources/models/Cargo.toml | 1 + sources/models/src/lib.rs | 53 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 826b31e5429..7a20ee712ae 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1758,6 +1758,7 @@ dependencies = [ "libc", "serde", "serde_json", + "serde_plain", "toml", ] diff --git a/sources/models/Cargo.toml b/sources/models/Cargo.toml index ce679bbbc6d..3d8cfa2ab16 100644 --- a/sources/models/Cargo.toml +++ b/sources/models/Cargo.toml @@ -14,6 +14,7 @@ bottlerocket-release.workspace = true libc.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +serde_plain.workspace = true toml.workspace = true # settings plugins diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 8dd88bc4e78..7692545d47e 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -25,6 +25,7 @@ use bottlerocket_release::BottlerocketRelease; use bottlerocket_settings_models::model_derive::model; use bottlerocket_settings_plugin::BottlerocketSettings; use serde::{Deserialize, Serialize}; +use serde_plain::derive_fromstr_from_deserialize; use std::collections::HashMap; use bottlerocket_settings_models::modeled_types::SingleLineString; @@ -87,3 +88,55 @@ struct Report { name: String, description: String, } + +/// Strength represents whether the settings can be overridden by a setting generator or not. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum Strength { + Strong, + Weak, +} + +impl std::fmt::Display for Strength { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Strength::Strong => write!(f, "strong"), + Strength::Weak => write!(f, "weak"), + } + } +} + +derive_fromstr_from_deserialize!(Strength); + +/// Struct to hold the setting generator definition containing +/// command, strength, skip-if-populated +#[derive(Deserialize, Serialize, std::fmt::Debug, PartialEq)] +pub struct SettingsGenerator { + pub command: String, + pub strength: Strength, + #[serde(rename = "skip-if-populated")] + pub skip_if_populated: bool, +} + +impl SettingsGenerator { + pub fn is_weak(&self) -> bool { + self.strength == Strength::Weak + } + + pub fn with_command(command: String) -> Self { + SettingsGenerator { + command, + ..SettingsGenerator::default() + } + } +} + +impl Default for SettingsGenerator { + fn default() -> Self { + SettingsGenerator { + command: String::new(), + strength: Strength::Strong, + skip_if_populated: true, + } + } +} From 979699de0ebeb07a847a9a5d5f25198ff2900cc3 Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Sun, 17 Nov 2024 23:03:12 +0000 Subject: [PATCH 06/12] common_migrations: add new migrations to handle settings-generator as struct - RemoveWeakSettingsMigration: When we downgrade multiple version to a version where migrator is not aware of deleting the setting-generator as struct or the strength file. This migration will help us to delete the strength and setting generator metadata. - MetadataStringToStructMigration: migration to change host-containers source metadata to struct from string. - ReplaceSettingWithSettingGeneratorMigration: Migration to replace a setting with setting generator. This migration will be used to convert a strong setting to a weak setting. --- sources/Cargo.lock | 1 + .../migration/migration-helpers/Cargo.toml | 1 + .../src/common_migrations.rs | 400 +++++++++++++++++- .../migration-helpers/src/datastore_helper.rs | 4 +- .../migration/migration-helpers/src/error.rs | 6 + 5 files changed, 409 insertions(+), 3 deletions(-) diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 7a20ee712ae..3661f1c8e95 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1701,6 +1701,7 @@ dependencies = [ "datastore", "handlebars", "maplit", + "models", "schnauzer", "serde", "serde_json", diff --git a/sources/api/migration/migration-helpers/Cargo.toml b/sources/api/migration/migration-helpers/Cargo.toml index 0464988bd28..3a333e43d45 100644 --- a/sources/api/migration/migration-helpers/Cargo.toml +++ b/sources/api/migration/migration-helpers/Cargo.toml @@ -12,6 +12,7 @@ exclude = ["README.md"] bottlerocket-release.workspace = true datastore.workspace = true handlebars.workspace = true +models.workspace = true schnauzer.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/sources/api/migration/migration-helpers/src/common_migrations.rs b/sources/api/migration/migration-helpers/src/common_migrations.rs index 90056cce256..fbc677d2d8c 100644 --- a/sources/api/migration/migration-helpers/src/common_migrations.rs +++ b/sources/api/migration/migration-helpers/src/common_migrations.rs @@ -1,9 +1,10 @@ use crate::{error, Migration, MigrationData, Result}; use schnauzer::import::{json_settings::JsonSettingsResolver, StaticHelperResolver}; use serde::Serialize; +use serde_json::{Map, Value}; use shlex::Shlex; use snafu::{OptionExt, ResultExt}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// We use this migration when we add settings and want to make sure they're removed before we go /// back to old versions that don't understand them. @@ -261,6 +262,110 @@ impl Migration for ReplaceStringMigration { // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +/// We use this migration to replace a setting string with object setting generator. +pub struct ReplaceSettingWithSettingGeneratorMigration { + pub setting: &'static str, + pub old_val: &'static str, + pub new_val: &'static str, + pub setting_gen_command: &'static str, + pub skip_if_populated: bool, +} + +impl Migration for ReplaceSettingWithSettingGeneratorMigration { + fn forward(&mut self, mut input: MigrationData) -> Result { + if let Some(data) = input.data.get_mut(self.setting) { + match data { + serde_json::Value::String(data) => { + if data == self.old_val { + self.new_val.clone_into(data); + println!( + "Changed value of '{}' from '{}' to '{}' on upgrade", + self.setting, self.old_val, self.new_val + ); + + let metadata = input.metadata.entry(self.setting.to_string()).or_default(); + let mut metadata_value = Map::new(); + metadata_value.insert( + "command".to_string(), + serde_json::Value::String(self.setting_gen_command.to_string()), + ); + metadata_value.insert( + "strength".to_string(), + serde_json::Value::String("weak".to_string()), + ); + metadata_value.insert( + "skip-if-populated".to_string(), + serde_json::Value::Bool(self.skip_if_populated), + ); + + metadata.insert( + "setting-generator".to_owned(), + serde_json::Value::Object(metadata_value), + ); + + metadata.insert( + "strength".to_owned(), + serde_json::Value::String("weak".to_string()), + ); + } else { + println!( + "'{}' is not set to '{}', leaving alone", + self.setting, self.old_val + ); + } + } + _ => { + println!( + "'{}' is set to non-string value '{}'; ReplaceSettingWithSettingGeneratorMigration expects a string setting value", + self.setting, data + ); + } + } + } else { + println!("Found no '{}' to change on upgrade", self.setting); + } + Ok(input) + } + + fn backward(&mut self, mut input: MigrationData) -> Result { + if let Some(data) = input.data.get_mut(self.setting) { + match data { + serde_json::Value::String(data) => { + if data == self.new_val { + self.old_val.clone_into(data); + println!( + "Changed value of '{}' from '{}' to '{}' on downgrade", + self.setting, self.new_val, self.old_val + ); + let metadata = input.metadata.entry(self.setting.to_string()).or_default(); + metadata.remove("setting-generator"); + metadata.remove("strength"); + } else { + println!( + "'{}' is not set to '{}', leaving alone", + self.setting, self.new_val + ); + } + } + _ => { + println!( + "'{}' is set to non-string value '{}'; ReplaceSettingWithSettingGeneratorMigration expects a string setting value", + self.setting, data + ); + } + } + } else { + input.data.insert( + self.setting.to_owned(), + serde_json::Value::String(self.old_val.to_string()), + ); + } + Ok(input) + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + /// We use this migration when we need to replace settings that contain lists of string values; /// for example, when a release changes the list of configuration-files associated with a service. // String is the only type we use today, and handling multiple value types is more complicated than @@ -1157,6 +1262,246 @@ mod test_replace_schnauzer_migration { // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +// We use this migration to change the aws host-containers setting-generator to struct from string. +// A metadata strength as `weak`, shows that associated setting is a weak setting. +pub struct MetadataStringToStructMigration { + pub setting: &'static str, + pub old_cmdline: &'static str, + pub new_cmdline: &'static str, + pub metadata_type: &'static str, + pub binary_for_generator: &'static str, + pub skip_if_populated: bool, +} + +impl MetadataStringToStructMigration { + fn get_setting_cmdline(&self, input: &MigrationData, direction: &str) -> Option { + input + .metadata + .get(self.setting) + .or_else(|| { + eprintln!("'{}' has no metadata", self.setting); + None + }) + .and_then(|metadata| { + // Get metadata type i.e setting-generator + let metadata_type = metadata.get(self.metadata_type); + if metadata_type.is_none() { + eprintln!( + "'{}' has no {} key in metadata", + self.setting, self.metadata_type + ); + } + metadata_type + }) + .and_then(|metadata_value| { + // When going backward the setting generator in the input is as a struct + // {command: "schnauzer-v2", strength: "weak", skip_if_populated: false} + // When going forward the setting generator in the input is as a string + // "schnauzer-v2" + let metadata_command = if direction.eq("backward") { + metadata_value["command"].as_str() + } else { + metadata_value.as_str() + }; + + if metadata_command.is_none() { + eprintln!( + "'{}' has invalid setting-generator command value '{}'", + self.setting, metadata_value + ); + } + metadata_command + }) + .and_then(|metadata_cmdline| { + // We are checking with command for metadata starts with the given binary + // i.e. schnauzer for host containers metadata i.e setting-generator + if !metadata_cmdline + .trim() + .starts_with(self.binary_for_generator) + { + eprintln!( + "'{}' has {} setting-generator value '{}'", + self.setting, self.binary_for_generator, metadata_cmdline, + ); + None + } else { + Some(metadata_cmdline.to_string()) + } + }) + } + + fn update_commandline_to_struct( + &self, + current_schnauzer_cmdline: &str, + outgoing_setting_data: &str, + outgoing_schnauzer_cmdline: &str, + incoming_schnauzer_cmdline: &str, + input: &mut MigrationData, + direction: &str, + ) -> Result<()> { + if current_schnauzer_cmdline == outgoing_schnauzer_cmdline { + // Update the metadata to struct for forward direction and to string for backward direction + let metadata = input.metadata.entry(self.setting.to_string()).or_default(); + if direction.eq("backward") { + metadata.insert( + self.metadata_type.to_string(), + serde_json::Value::String(incoming_schnauzer_cmdline.to_string()), + ); + } else { + // Create the Map for the value of metadata + let mut metadata_value = Map::new(); + metadata_value.insert( + "command".to_string(), + serde_json::Value::String(self.new_cmdline.to_string()), + ); + metadata_value.insert( + "strength".to_string(), + serde_json::Value::String("weak".to_string()), + ); + metadata_value.insert( + "skip-if-populated".to_string(), + serde_json::Value::Bool(self.skip_if_populated), + ); + + metadata.insert( + self.metadata_type.to_owned(), + serde_json::Value::Object(metadata_value), + ); + } + + let input_data = structure_migration_data_for_templates(&input.data)?; + let input_data = + serde_json::to_value(input_data).context(error::SerializeTemplateDataSnafu)?; + + // Generate settings data using the setting's outgoing template so we can confirm + // it matches our expected value; if not, the user has changed it and we should stop. + let template_importer = SchnauzerMigrationTemplateImporter::new(input_data); + let outgoing_command_args = Shlex::new(outgoing_schnauzer_cmdline); + + let tokio_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .context(error::CreateTokioRuntimeSnafu)?; + + let generated_old_data = tokio_runtime + .block_on(async { + schnauzer::v2::cli::run_with_args(outgoing_command_args, &template_importer) + .await + }) + .with_context(|_| error::RenderSchnauzerV2TemplateSnafu { + cmdline: outgoing_schnauzer_cmdline.to_string(), + })?; + + if generated_old_data == *outgoing_setting_data { + // Generate settings data using the setting's incoming template + let incoming_command_args = Shlex::new(incoming_schnauzer_cmdline); + let generated_new_data = tokio_runtime + .block_on(async { + schnauzer::v2::cli::run_with_args(incoming_command_args, &template_importer) + .await + }) + .with_context(|_| error::RenderSchnauzerV2TemplateSnafu { + cmdline: incoming_schnauzer_cmdline.to_string(), + })?; + println!( + "Changing value of '{}' from '{}' to '{}'", + self.setting, outgoing_setting_data, generated_new_data + ); + // Update settings value with new generated value + input.data.insert( + self.setting.to_string(), + serde_json::Value::String(generated_new_data.clone()), + ); + + if direction.eq("forward") { + metadata.insert("strength".to_string(), Value::String("weak".to_string())); + } else { + metadata.remove("strength"); + } + } else { + println!( + "'{}' is not set to '{}', leaving alone", + self.setting, generated_old_data + ); + } + } + + Ok(()) + } +} + +impl Migration for MetadataStringToStructMigration { + fn forward(&mut self, mut input: MigrationData) -> Result { + if let Some(input_value) = input.data.get(self.setting) { + let data = input_value + .as_str() + .context(error::NonStringSettingDataTypeSnafu { + setting: self.setting, + })?; + + if let Some(setting_cmdline) = &self.get_setting_cmdline(&input, "forward") { + if setting_cmdline == self.old_cmdline { + self.update_commandline_to_struct( + setting_cmdline, + // Clone the input string; we need to give the function mutable access to + // the structure that contains the string, so we can't pass a reference into the + // structure. + #[allow(clippy::unnecessary_to_owned)] + &data.to_owned(), + self.old_cmdline, + self.new_cmdline, + &mut input, + "forward", + )?; + } else { + println!( + "Generator for '{}' is not set to '{}', leaving alone", + self.setting, self.old_cmdline + ); + } + } + } else { + println!("Found no '{}' to change on upgrade", self.setting); + } + + Ok(input) + } + + fn backward(&mut self, mut input: MigrationData) -> Result { + if let Some(input_value) = input.data.get(self.setting) { + let data = input_value + .as_str() + .context(error::NonStringSettingDataTypeSnafu { + setting: self.setting, + })?; + println!( + "Updating schnauzer template and value of '{}' on downgrade", + self.setting + ); + if let Some(setting_cmdline) = &self.get_setting_cmdline(&input, "backward") { + self.update_commandline_to_struct( + setting_cmdline, + // Clone the input string; we need to give the function mutable access to + // the structure that contains the string, so we can't pass a reference into the + // structure. + #[allow(clippy::unnecessary_to_owned)] + &data.to_owned(), + self.new_cmdline, + self.old_cmdline, + &mut input, + "backward", + )?; + } + } else { + println!("Found no '{}' to change on downgrade", self.setting); + } + + Ok(input) + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + /// We use this migration when we add metadata and want to make sure they're removed before we go /// back to old versions that don't understand them. #[derive(Debug)] @@ -1914,3 +2259,56 @@ impl Migration for NoOpMigration { Ok(input) } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +// When we downgrade multiple version to a version where migrator is not aware of deleting the +// setting-generator as struct or the strength file. +// This migration will remove the setting-generator as struct and strength metadata. +#[derive(Debug)] +pub struct RemoveWeakSettingsMigration; + +impl Migration for RemoveWeakSettingsMigration { + /// No work to do on forward migrations, copy the same datastore + fn forward(&mut self, input: MigrationData) -> Result { + println!("RemoveWeakSettingsMigration has no work to do on upgrade.",); + Ok(input) + } + + /// No work to do on backward migrations, copy the same datastore + fn backward(&mut self, mut input: MigrationData) -> Result { + let mut keys_to_remove = HashSet::new(); + // Collect keys where the inner HashMap contains the key "strength" + for (key, inner_map) in &input.metadata { + if let Some(strength) = inner_map.get("strength") { + if strength == &Value::String("weak".to_string()) { + keys_to_remove.insert(key.clone()); + } + } + } + // Remove strength metadata for weak settings + for key in keys_to_remove { + let metadata = input.metadata.get(&key); + if let Some(metadata) = metadata { + let mut inner_map = metadata.clone(); + inner_map.remove("strength"); + input.metadata.insert(key.clone(), inner_map); + } + input.data.remove(&key); + } + + // Remove all the setting generators with weak strength + for (key, inner_map) in input.metadata.clone() { + if let Some(Value::Object(_)) = inner_map.get("setting-generator") { + // We need to remove the setting-generators that are an object + // because the API in destination version is not aware about the + // object setting generator. + let mut inner_map = inner_map.clone(); + inner_map.remove("setting-generator"); + input.metadata.insert(key.clone(), inner_map); + } + } + + Ok(input) + } +} diff --git a/sources/api/migration/migration-helpers/src/datastore_helper.rs b/sources/api/migration/migration-helpers/src/datastore_helper.rs index cfb31685ebb..f2986d8a385 100644 --- a/sources/api/migration/migration-helpers/src/datastore_helper.rs +++ b/sources/api/migration/migration-helpers/src/datastore_helper.rs @@ -53,7 +53,7 @@ pub(crate) fn get_input_data( let mut metadata = HashMap::new(); if let Committed::Live = committed { let raw_metadata = datastore - .get_metadata_prefix("", &None as &Option<&str>) + .get_metadata_prefix("", committed, &None as &Option<&str>) .context(error::GetMetadataSnafu)?; for (data_key, meta_map) in raw_metadata.into_iter() { // See notes above about storing key Strings and Values. @@ -114,7 +114,7 @@ pub(crate) fn set_output_data( })?; let value = serialize_scalar(&raw_value).context(error::SerializeSnafu)?; datastore - .set_metadata(&metadata_key, &data_key, value) + .set_metadata(&metadata_key, &data_key, value, committed) .context(error::DataStoreWriteSnafu)?; } } diff --git a/sources/api/migration/migration-helpers/src/error.rs b/sources/api/migration/migration-helpers/src/error.rs index 2abd5bc62a3..04e5042ec5b 100644 --- a/sources/api/migration/migration-helpers/src/error.rs +++ b/sources/api/migration/migration-helpers/src/error.rs @@ -131,6 +131,12 @@ pub enum Error { #[snafu(display("Failed to create async runtime: {}", source))] CreateTokioRuntime { source: std::io::Error }, + + #[snafu(display( + "Error in deserializing response value to SettingsGenerator: {}", + source + ))] + DeserializeSettingsGenerator { source: serde_json::Error }, } /// Result alias containing our Error type. From 535f5394a1c395856529a043a490a97f92763ca9 Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 6 Dec 2024 00:51:43 +0000 Subject: [PATCH 07/12] migrations: Change public control host-container source metadata to object This migration will change the public control host-container source to setting-generator metadata(that will generate source using sundog) on forward migration. In backward migration, it will reset the value of the source as string. --- Release.toml | 5 ++++ sources/Cargo.lock | 14 ++++++++++ sources/Cargo.toml | 2 ++ .../Cargo.toml | 15 +++++++++++ .../src/main.rs | 27 +++++++++++++++++++ 5 files changed, 63 insertions(+) create mode 100644 sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/Cargo.toml create mode 100644 sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/src/main.rs diff --git a/Release.toml b/Release.toml index c20c239dd20..c40275d845e 100644 --- a/Release.toml +++ b/Release.toml @@ -379,3 +379,8 @@ version = "1.28.0" "migrate_v1.28.0_aws-control-container-v0-7-18.lz4", "migrate_v1.28.0_public-control-container-v0-7-18.lz4", ] +"(1.28.0, 1.29.0)" = [ + "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4", + "migrate_v1.29.0_change-public-control-container-to-set-gen.lz4", +] + diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 3661f1c8e95..806d9887624 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -774,6 +774,20 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "change-public-admin-container-to-set-gen" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + +[[package]] +name = "change-public-control-container-to-set-gen" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "clang-sys" version = "1.8.1" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index d59c84dfe04..0aade74641a 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -71,6 +71,8 @@ members = [ "settings-migrations/v1.28.0/public-admin-container-v0-11-14", "settings-migrations/v1.28.0/aws-control-container-v0-7-18", "settings-migrations/v1.28.0/public-control-container-v0-7-18", + "settings-migrations/v1.29.0/change-public-admin-container-to-set-gen", + "settings-migrations/v1.29.0/change-public-control-container-to-set-gen", "settings-plugins/aws-dev", "settings-plugins/aws-ecs-1", diff --git a/sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/Cargo.toml b/sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/Cargo.toml new file mode 100644 index 00000000000..52f62755d02 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "change-public-control-container-to-set-gen" +version = "0.1.0" +authors = ["Shikha Vyaghra "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +migration-helpers.workspace = true diff --git a/sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/src/main.rs b/sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/src/main.rs new file mode 100644 index 00000000000..399292ab43f --- /dev/null +++ b/sources/settings-migrations/v1.29.0/change-public-control-container-to-set-gen/src/main.rs @@ -0,0 +1,27 @@ +use migration_helpers::common_migrations::ReplaceSettingWithSettingGeneratorMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +const OLD_CONTROL_CTR: &str = "public.ecr.aws/bottlerocket/bottlerocket-control:v0.7.18"; +const NEW_CONTROL_CTR: &str = "public.ecr.aws/bottlerocket/bottlerocket-control:v0.7.18"; + +/// We bumped the version of the default admin container +fn run() -> Result<()> { + migrate(ReplaceSettingWithSettingGeneratorMigration { + setting: "settings.host-containers.control.source", + old_val: OLD_CONTROL_CTR, + new_val: NEW_CONTROL_CTR, + setting_gen_command: "emitter public.ecr.aws/bottlerocket/bottlerocket-control:v0.7.17", + skip_if_populated: true, + }) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} From 3323f8efba86d7f33540c477f12aa3edffc8280e Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 6 Dec 2024 00:57:07 +0000 Subject: [PATCH 08/12] migrations: Change aws admin host-container source metadata to object This migration will change the aws admin host-container source setting-generator from string to object setting-generator metadata (that will generate source using sundog) on forward migration. We will also add a strength file with value "weak". In backward migration, it will reset the value of the source setting generator as string and delete the strength metadata. --- Release.toml | 1 + sources/Cargo.lock | 7 +++++ sources/Cargo.toml | 1 + .../Cargo.toml | 15 ++++++++++ .../src/main.rs | 30 +++++++++++++++++++ 5 files changed, 54 insertions(+) create mode 100644 sources/settings-migrations/v1.29.0/update-settings-generator-admin/Cargo.toml create mode 100644 sources/settings-migrations/v1.29.0/update-settings-generator-admin/src/main.rs diff --git a/Release.toml b/Release.toml index c40275d845e..0d39eae7481 100644 --- a/Release.toml +++ b/Release.toml @@ -380,6 +380,7 @@ version = "1.28.0" "migrate_v1.28.0_public-control-container-v0-7-18.lz4", ] "(1.28.0, 1.29.0)" = [ + "migrate_v1.29.0_update-settings-generator-admin.lz4", "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4", "migrate_v1.29.0_change-public-control-container-to-set-gen.lz4", ] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 806d9887624..a5790ed53ed 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -3594,6 +3594,13 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "update-settings-generator-admin" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "url" version = "2.5.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 0aade74641a..7bbeea64eb2 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -71,6 +71,7 @@ members = [ "settings-migrations/v1.28.0/public-admin-container-v0-11-14", "settings-migrations/v1.28.0/aws-control-container-v0-7-18", "settings-migrations/v1.28.0/public-control-container-v0-7-18", + "settings-migrations/v1.29.0/update-settings-generator-admin", "settings-migrations/v1.29.0/change-public-admin-container-to-set-gen", "settings-migrations/v1.29.0/change-public-control-container-to-set-gen", diff --git a/sources/settings-migrations/v1.29.0/update-settings-generator-admin/Cargo.toml b/sources/settings-migrations/v1.29.0/update-settings-generator-admin/Cargo.toml new file mode 100644 index 00000000000..0348a696570 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/update-settings-generator-admin/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "update-settings-generator-admin" +version = "0.1.0" +authors = ["Shikha Vyaghra "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +migration-helpers.workspace = true diff --git a/sources/settings-migrations/v1.29.0/update-settings-generator-admin/src/main.rs b/sources/settings-migrations/v1.29.0/update-settings-generator-admin/src/main.rs new file mode 100644 index 00000000000..5b4fcf8b609 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/update-settings-generator-admin/src/main.rs @@ -0,0 +1,30 @@ +use migration_helpers::common_migrations::MetadataStringToStructMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +const OLD_ADMIN_CTR_CMDLINE: &str = + "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-admin:v0.11.14'"; +const NEW_ADMIN_CTR_CMDLINE: &str = + "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-admin:v0.11.14'"; + +/// We bumped the version of the default admin container +fn run() -> Result<()> { + migrate(MetadataStringToStructMigration { + setting: "settings.host-containers.admin.source", + old_cmdline: OLD_ADMIN_CTR_CMDLINE, + new_cmdline: NEW_ADMIN_CTR_CMDLINE, + metadata_type: "setting-generator", + binary_for_generator: "schnauzer-v2", + skip_if_populated: true, + }) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} From 7356b2c87f9ddb32919334024aae6a29c2c177d4 Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 6 Dec 2024 01:03:41 +0000 Subject: [PATCH 09/12] migrations: Change public control host-container source metadata to object This migration will change the public control host-container source to setting-generator metadata(that will generate source using sundog) on forward migration. In backward migration, it will reset the value of the source as string. --- Release.toml | 1 + sources/Cargo.lock | 7 +++++ sources/Cargo.toml | 1 + .../Cargo.toml | 15 ++++++++++ .../src/main.rs | 30 +++++++++++++++++++ 5 files changed, 54 insertions(+) create mode 100644 sources/settings-migrations/v1.29.0/update-settings-generator-control/Cargo.toml create mode 100644 sources/settings-migrations/v1.29.0/update-settings-generator-control/src/main.rs diff --git a/Release.toml b/Release.toml index 0d39eae7481..37c0907a18c 100644 --- a/Release.toml +++ b/Release.toml @@ -381,6 +381,7 @@ version = "1.28.0" ] "(1.28.0, 1.29.0)" = [ "migrate_v1.29.0_update-settings-generator-admin.lz4", + "migrate_v1.29.0_update-settings-generator-control.lz4", "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4", "migrate_v1.29.0_change-public-control-container-to-set-gen.lz4", ] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index a5790ed53ed..dfdb4b4cf11 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -3601,6 +3601,13 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "update-settings-generator-control" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "url" version = "2.5.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 7bbeea64eb2..c147bf192b8 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -72,6 +72,7 @@ members = [ "settings-migrations/v1.28.0/aws-control-container-v0-7-18", "settings-migrations/v1.28.0/public-control-container-v0-7-18", "settings-migrations/v1.29.0/update-settings-generator-admin", + "settings-migrations/v1.29.0/update-settings-generator-control", "settings-migrations/v1.29.0/change-public-admin-container-to-set-gen", "settings-migrations/v1.29.0/change-public-control-container-to-set-gen", diff --git a/sources/settings-migrations/v1.29.0/update-settings-generator-control/Cargo.toml b/sources/settings-migrations/v1.29.0/update-settings-generator-control/Cargo.toml new file mode 100644 index 00000000000..490b44b9a6c --- /dev/null +++ b/sources/settings-migrations/v1.29.0/update-settings-generator-control/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "update-settings-generator-control" +version = "0.1.0" +authors = ["Shikha Vyaghra "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +migration-helpers.workspace = true diff --git a/sources/settings-migrations/v1.29.0/update-settings-generator-control/src/main.rs b/sources/settings-migrations/v1.29.0/update-settings-generator-control/src/main.rs new file mode 100644 index 00000000000..e9932990ce4 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/update-settings-generator-control/src/main.rs @@ -0,0 +1,30 @@ +use migration_helpers::common_migrations::MetadataStringToStructMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +const OLD_CONTROL_CTR_CMDLINE: &str = + "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-control:v0.7.18'"; +const NEW_CONTROL_CTR_CMDLINE: &str = + "schnauzer-v2 render --requires 'aws@v1(helpers=[ecr-prefix])' --template '{{ ecr-prefix settings.aws.region }}/bottlerocket-control:v0.7.18'"; + +/// We bumped the version of the default control container +fn run() -> Result<()> { + migrate(MetadataStringToStructMigration { + setting: "settings.host-containers.control.source", + old_cmdline: OLD_CONTROL_CTR_CMDLINE, + new_cmdline: NEW_CONTROL_CTR_CMDLINE, + metadata_type: "setting-generator", + binary_for_generator: "schnauzer-v2", + skip_if_populated: true, + }) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} From e787092faf9e8e5828c66062d1f82aec083cd7bb Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 6 Dec 2024 01:10:57 +0000 Subject: [PATCH 10/12] migrations: remove all the weak setting on downgrade We need this migration that will delete all the weak settings and setting-generators. --- Release.toml | 1 + sources/Cargo.lock | 7 +++++++ sources/Cargo.toml | 1 + .../remove-weak-settings-migration/Cargo.toml | 15 +++++++++++++++ .../remove-weak-settings-migration/src/main.rs | 18 ++++++++++++++++++ 5 files changed, 42 insertions(+) create mode 100644 sources/settings-migrations/v1.29.0/remove-weak-settings-migration/Cargo.toml create mode 100644 sources/settings-migrations/v1.29.0/remove-weak-settings-migration/src/main.rs diff --git a/Release.toml b/Release.toml index 37c0907a18c..099c26bf3db 100644 --- a/Release.toml +++ b/Release.toml @@ -380,6 +380,7 @@ version = "1.28.0" "migrate_v1.28.0_public-control-container-v0-7-18.lz4", ] "(1.28.0, 1.29.0)" = [ + "migrate_v1.29.0_remove_weak_settings_migration.lz4", "migrate_v1.29.0_update-settings-generator-admin.lz4", "migrate_v1.29.0_update-settings-generator-control.lz4", "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4", diff --git a/sources/Cargo.lock b/sources/Cargo.lock index dfdb4b4cf11..02d438ca893 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -2265,6 +2265,13 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "remove-weak-settings-migration" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "repr_offset" version = "0.2.2" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index c147bf192b8..8030037b146 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -71,6 +71,7 @@ members = [ "settings-migrations/v1.28.0/public-admin-container-v0-11-14", "settings-migrations/v1.28.0/aws-control-container-v0-7-18", "settings-migrations/v1.28.0/public-control-container-v0-7-18", + "settings-migrations/v1.29.0/remove-weak-settings-migration", "settings-migrations/v1.29.0/update-settings-generator-admin", "settings-migrations/v1.29.0/update-settings-generator-control", "settings-migrations/v1.29.0/change-public-admin-container-to-set-gen", diff --git a/sources/settings-migrations/v1.29.0/remove-weak-settings-migration/Cargo.toml b/sources/settings-migrations/v1.29.0/remove-weak-settings-migration/Cargo.toml new file mode 100644 index 00000000000..b3dfdfcebe1 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/remove-weak-settings-migration/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "remove-weak-settings-migration" +version = "0.1.0" +authors = ["Shikha Vyaghra "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +migration-helpers.workspace = true diff --git a/sources/settings-migrations/v1.29.0/remove-weak-settings-migration/src/main.rs b/sources/settings-migrations/v1.29.0/remove-weak-settings-migration/src/main.rs new file mode 100644 index 00000000000..28572e95f1d --- /dev/null +++ b/sources/settings-migrations/v1.29.0/remove-weak-settings-migration/src/main.rs @@ -0,0 +1,18 @@ +use migration_helpers::common_migrations::RemoveWeakSettingsMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +// Remove the weak settings on downgrade +fn run() -> Result<()> { + migrate(RemoveWeakSettingsMigration) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} From a5bcf5625c14a8d074e43f1dd519612c0a86a0fb Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 6 Dec 2024 00:44:03 +0000 Subject: [PATCH 11/12] migrations: Change public admin host-container source metadata to object This migration will change the public admin host-container source to setting-generator metadata(that will generate source using sundog) on forward migration. In backward migration, it will reset the value of the source as string. --- Release.toml | 2 -- .../Cargo.toml | 15 +++++++++++ .../src/main.rs | 27 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/Cargo.toml create mode 100644 sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/src/main.rs diff --git a/Release.toml b/Release.toml index 099c26bf3db..06bfeff0539 100644 --- a/Release.toml +++ b/Release.toml @@ -379,11 +379,9 @@ version = "1.28.0" "migrate_v1.28.0_aws-control-container-v0-7-18.lz4", "migrate_v1.28.0_public-control-container-v0-7-18.lz4", ] -"(1.28.0, 1.29.0)" = [ "migrate_v1.29.0_remove_weak_settings_migration.lz4", "migrate_v1.29.0_update-settings-generator-admin.lz4", "migrate_v1.29.0_update-settings-generator-control.lz4", "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4", "migrate_v1.29.0_change-public-control-container-to-set-gen.lz4", -] diff --git a/sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/Cargo.toml b/sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/Cargo.toml new file mode 100644 index 00000000000..de0a02f71a3 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "change-public-admin-container-to-set-gen" +version = "0.1.0" +authors = ["Shikha Vyaghra "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +migration-helpers.workspace = true diff --git a/sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/src/main.rs b/sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/src/main.rs new file mode 100644 index 00000000000..32768ad7ec1 --- /dev/null +++ b/sources/settings-migrations/v1.29.0/change-public-admin-container-to-set-gen/src/main.rs @@ -0,0 +1,27 @@ +use migration_helpers::common_migrations::ReplaceSettingWithSettingGeneratorMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +const OLD_ADMIN_CTR: &str = "public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.14"; +const NEW_ADMIN_CTR: &str = "public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.14"; + +/// We bumped the version of the default admin container +fn run() -> Result<()> { + migrate(ReplaceSettingWithSettingGeneratorMigration { + setting: "settings.host-containers.admin.source", + old_val: OLD_ADMIN_CTR, + new_val: NEW_ADMIN_CTR, + setting_gen_command: "emitter public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.13", + skip_if_populated: true, + }) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} From b77efc3965d39d7cb145bbecd0a5db1b6e2926ea Mon Sep 17 00:00:00 2001 From: Shikha Vyaghra Date: Fri, 6 Dec 2024 18:56:16 +0000 Subject: [PATCH 12/12] release: update Bottlerocket release version to 1.29.0 --- Release.toml | 4 +++- Twoliter.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Release.toml b/Release.toml index 06bfeff0539..e2a7569a707 100644 --- a/Release.toml +++ b/Release.toml @@ -1,4 +1,4 @@ -version = "1.28.0" +version = "1.29.0" [migrations] "(0.3.1, 0.3.2)" = ["migrate_v0.3.2_admin-container-v0-5-0.lz4"] @@ -379,9 +379,11 @@ version = "1.28.0" "migrate_v1.28.0_aws-control-container-v0-7-18.lz4", "migrate_v1.28.0_public-control-container-v0-7-18.lz4", ] +"(1.28.0, 1.29.0)" = [ "migrate_v1.29.0_remove_weak_settings_migration.lz4", "migrate_v1.29.0_update-settings-generator-admin.lz4", "migrate_v1.29.0_update-settings-generator-control.lz4", "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4", "migrate_v1.29.0_change-public-control-container-to-set-gen.lz4", +] diff --git a/Twoliter.toml b/Twoliter.toml index 9439fed452f..5e8f7efb82a 100644 --- a/Twoliter.toml +++ b/Twoliter.toml @@ -1,5 +1,5 @@ schema-version = 1 -release-version = "1.28.0" +release-version = "1.29.0" [vendor.bottlerocket] registry = "public.ecr.aws/bottlerocket"