Remove host container migration

Release.toml

@@ -1,4 +1,4 @@
-version = "1.28.0"
+version = "1.29.0"
 
 [migrations]
 "(0.3.1, 0.3.2)" = ["migrate_v0.3.2_admin-container-v0-5-0.lz4"]
@@ -379,3 +379,11 @@ version = "1.28.0"
     "migrate_v1.28.0_aws-control-container-v0-7-18.lz4",
     "migrate_v1.28.0_public-control-container-v0-7-18.lz4",
 ]
+"(1.28.0, 1.29.0)" = [
+    "migrate_v1.29.0_remove_weak_settings_migration.lz4",
+    "migrate_v1.29.0_update-settings-generator-admin.lz4",
+    "migrate_v1.29.0_update-settings-generator-control.lz4",
+    "migrate_v1.29.0_change-public-admin-container-to-set-gen.lz4",
+    "migrate_v1.29.0_change-public-control-container-to-set-gen.lz4",
+]
+

Twoliter.toml

@@ -1,5 +1,5 @@
 schema-version = 1
-release-version = "1.28.0"
+release-version = "1.29.0"
 
 [vendor.bottlerocket]
 registry = "public.ecr.aws/bottlerocket"

sources/Cargo.toml

@@ -71,6 +71,11 @@ members = [
     "settings-migrations/v1.28.0/public-admin-container-v0-11-14",
     "settings-migrations/v1.28.0/aws-control-container-v0-7-18",
     "settings-migrations/v1.28.0/public-control-container-v0-7-18",
+    "settings-migrations/v1.29.0/remove-weak-settings-migration",
+    "settings-migrations/v1.29.0/update-settings-generator-admin",
+    "settings-migrations/v1.29.0/update-settings-generator-control",
+    "settings-migrations/v1.29.0/change-public-admin-container-to-set-gen",
+    "settings-migrations/v1.29.0/change-public-control-container-to-set-gen",
 
     "settings-plugins/aws-dev",
     "settings-plugins/aws-ecs-1",

sources/api/apiclient/README.md

@@ -174,6 +174,10 @@ You can see all your pending settings like this:
 ```shell
 apiclient raw -u /tx
 ```
+You can also see pending metadata along with pending setting using version 2 of `/tx` like this:
+```shell
+apiclient raw -u /v2/tx
+```
 
 To *commit* the settings, and let the system apply them to any relevant configuration files or services, do this:
 ```shell

sources/api/apiclient/README.tpl

@@ -174,6 +174,10 @@ You can see all your pending settings like this:
 ```shell
 apiclient raw -u /tx
 ```
+You can also see pending metadata along with pending setting using version 2 of `/tx` like this:
+```shell
+apiclient raw -u /v2/tx
+```
 
 To *commit* the settings, and let the system apply them to any relevant configuration files or services, do this:
 ```shell

sources/api/datastore/src/error.rs

@@ -59,6 +59,12 @@ pub enum Error {
 
     #[snafu(display("Key name beyond maximum length {}: {}", name, max))]
     KeyTooLong { name: String, max: usize },
+
+    #[snafu(display("Unable to serialize data: {}", source))]
+    Serialize { source: serde_json::Error },
+
+    #[snafu(display("Unable to de-serialize data: {}", source))]
+    DeSerialize { source: serde_json::Error },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;

sources/api/datastore/src/filesystem.rs

@@ -4,7 +4,7 @@
 //! Data is kept in files with paths resembling the keys, e.g. a/b/c for a.b.c, and metadata is
 //! kept in a suffixed file next to the data, e.g. a/b/c.meta for metadata "meta" about a.b.c
 
-use log::{debug, error, trace};
+use log::{debug, error, trace, warn};
 use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, HashSet};
@@ -13,6 +13,8 @@ use std::io;
 use std::path::{self, Path, PathBuf};
 use walkdir::{DirEntry, WalkDir};
 
+use crate::{deserialize_scalar, serialize_scalar, ScalarError};
+
 use super::key::{Key, KeyType};
 use super::{error, Committed, DataStore, Result};
 
@@ -413,14 +415,15 @@ impl DataStore for FilesystemDataStore {
     fn list_populated_metadata<S1, S2>(
         &self,
         prefix: S1,
+        committed: &Committed,
         metadata_key_name: &Option<S2>,
     ) -> Result<HashMap<Key, HashSet<Key>>>
     where
         S1: AsRef<str>,
         S2: AsRef<str>,
     {
         // Find metadata key paths on disk
-        let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, &Committed::Live)?;
+        let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, committed)?;
 
         // For each file on disk, check the user's conditions, and add it to our output
         let mut result = HashMap::new();
@@ -460,8 +463,13 @@ impl DataStore for FilesystemDataStore {
         self.delete_key_path(path, committed)
     }
 
-    fn get_metadata_raw(&self, metadata_key: &Key, data_key: &Key) -> Result<Option<String>> {
-        let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?;
+    fn get_metadata_raw(
+        &self,
+        metadata_key: &Key,
+        data_key: &Key,
+        committed: &Committed,
+    ) -> Result<Option<String>> {
+        let path = self.metadata_path(metadata_key, data_key, committed)?;
         read_file_for_key(metadata_key, &path)
     }
 
@@ -470,8 +478,9 @@ impl DataStore for FilesystemDataStore {
         metadata_key: &Key,
         data_key: &Key,
         value: S,
+        committed: &Committed,
     ) -> Result<()> {
-        let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?;
+        let path = self.metadata_path(metadata_key, data_key, committed)?;
         write_file_mkdir(path, value)
     }
 
@@ -489,6 +498,95 @@ impl DataStore for FilesystemDataStore {
         let pending = Committed::Pending {
             tx: transaction.into(),
         };
+
+        // We will first commit metadata to correctly check that if a setting exist and
+        // its strength is strong, we will not change it to weak.
+        // Get metadata from pending transaction
+        let transaction_metadata =
+            self.get_metadata_prefix("settings.", &pending, &None as &Option<&str>)?;
+
+        trace!(
+            "commit_transaction: transaction_metadata: {:?}",
+            transaction_metadata
+        );
+
+        for (key, value) in transaction_metadata {
+            for (metadata_key, metadata_value) in value {
+                // For now we are only processing the strength metadata from pending
+                // transaction to live
+                if metadata_key.name() != "strength" {
+                    continue;
+                }
+
+                // strength in pending transaction
+                let pending_strength: String =
+                    deserialize_scalar::<_, ScalarError>(&metadata_value.clone())
+                        .with_context(|_| error::DeSerializeSnafu {})?;
+
+                // Get the setting strength in live
+                // get_metadata function returns Ok(None) in case strength does not exist
+                // We will consider this case as strength equals strong.
+                let committed_strength =
+                    match self.get_metadata(&metadata_key, &key, &Committed::Live) {
+                        Ok(Some(v)) => v,
+                        Ok(None) => "strong".to_string(),
+                        Err(_) => continue,
+                    };
+
+                // The get key funtion returns Ok(None) in case if the path does not exist
+                // and error if some path exist and some error occurred in fetching
+                // Hence we we will return error in case of error
+                // from get key function and continue to add/change to weak key
+                // if the value is None.
+                let value = self.get_key(&key, &Committed::Live)?;
+
+                trace!(
+                    "commit_transaction: key: {:?}, metadata_key: {:?}, metadata_value: {:?}",
+                    key.name(),
+                    metadata_key.name(),
+                    metadata_value
+                );
+
+                match (pending_strength.as_str(), committed_strength.as_str()) {
+                    ("weak", "strong") => {
+                        // Do not change from strong to weak if setting exists
+                        // otherwise commit strength metadata with value as "weak"
+                        if value.is_some() {
+                            warn!("Trying to change the strength from strong to weak for key: {}, Operation ignored", key.name());
+                            continue;
+                        } else {
+                            let met_value = serialize_scalar::<_, ScalarError>(&pending_strength)
+                                .with_context(|_| error::SerializeSnafu {})?;
+
+                            self.set_metadata(&metadata_key, &key, met_value, &Committed::Live)?;
+                        }
+                    }
+                    ("strong", "weak") => {
+                        let met_value = serialize_scalar::<_, ScalarError>(&pending_strength)
+                            .with_context(|_| error::SerializeSnafu {})?;
+                        self.set_metadata(&metadata_key, &key, met_value, &Committed::Live)?;
+                    }
+                    ("weak", "weak") => {
+                        trace!("The strength for setting {} is already weak", key.name());
+                        continue;
+                    }
+                    ("strong", "strong") => {
+                        trace!("The strength for setting {} is already strong", key.name());
+                        continue;
+                    }
+                    _ => {
+                        warn!(
+                            "The strength for setting {} is not one of weak or strong. Pending strength: {}, Committed strength: {} ",
+                            key.name(),
+                            pending_strength,
+                            committed_strength
+                        );
+                        continue;
+                    }
+                };
+            }
+        }
+
         // Get data for changed keys
         let pending_data = self.get_prefix("settings.", &pending)?;