Merge pull request #491 from amazonlinux/meta-defaults

[storewolf] Improve default metadata representation

workspaces/api/storewolf/defaults.toml

@@ -17,10 +17,8 @@ restart-commands = []
 path = "/etc/hostname"
 template-path = "/usr/share/templates/hostname"
 
-[[metadata]]
-key = "settings.hostname"
-md = "affected-services"
-val = ["hostname"]
+[metadata.settings.hostname]
+affected-services = ["hostname"]
 
 # Kubernetes.
 
@@ -48,30 +46,12 @@ template-path = "/usr/share/templates/kubernetes-ca-crt"
 path = "/etc/containerd/config.toml"
 template-path = "/usr/share/templates/containerd-config-toml"
 
-[[metadata]]
-key = "settings.kubernetes.max-pods"
-md = "setting-generator"
-val = "pluto max-pods"
-
-[[metadata]]
-key = "settings.kubernetes.cluster-dns-ip"
-md = "setting-generator"
-val = "pluto cluster-dns-ip"
-
-[[metadata]]
-key = "settings.kubernetes.node-ip"
-md = "setting-generator"
-val = "pluto node-ip"
-
-[[metadata]]
-key = "settings.kubernetes.pod-infra-container-image"
-md = "setting-generator"
-val = "pluto pod-infra-container-image"
-
-[[metadata]]
-key = "settings.kubernetes"
-md = "affected-services"
-val = ["kubernetes"]
+[metadata.settings.kubernetes]
+max-pods.setting-generator = "pluto max-pods"
+cluster-dns-ip.setting-generator = "pluto cluster-dns-ip"
+node-ip.setting-generator = "pluto node-ip"
+pod-infra-container-image.setting-generator = "pluto pod-infra-container-image"
+affected-services = ["kubernetes"]
 
 # Updog.
 
@@ -83,15 +63,9 @@ restart-commands = []
 path = "/etc/updog.toml"
 template-path = "/usr/share/templates/updog-toml"
 
-[[metadata]]
-key = "settings.updates"
-md = "affected-services"
-val = ["updog"]
-
-[[metadata]]
-key = "settings.updates.seed"
-md = "setting-generator"
-val = "bork seed"
+[metadata.settings.updates]
+affected-services = ["updog"]
+seed.setting-generator = "bork seed"
 
 # HostContainers
 
@@ -109,10 +83,8 @@ superpowered = false
 configuration-files = []
 restart-commands = ["/usr/bin/host-containers"]
 
-[[metadata]]
-key = "settings.host-containers"
-md = "affected-services"
-val = ["host-containers"]
+[metadata.settings.host-containers]
+affected-services = ["host-containers"]
 
 # NTP
 
@@ -127,7 +99,5 @@ restart-commands = ["/bin/systemctl try-reload-or-restart chronyd.service"]
 path = "/etc/chrony.conf"
 template-path = "/usr/share/templates/chrony-conf"
 
-[[metadata]]
-key = "settings.ntp"
-md = "affected-services"
-val = "[chronyd]"
+[metadata.settings.ntp]
+affected-services = ["chronyd"]

workspaces/api/storewolf/src/main.rs

@@ -13,8 +13,9 @@ extern crate log;
 
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 use simplelog::{Config as LogConfig, LevelFilter, TermLogger, TerminalMode};
-use snafu::{OptionExt, ResultExt};
+use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, HashSet};
+use std::convert::TryFrom;
 use std::io;
 use std::os::unix::fs::symlink;
 use std::path::Path;
@@ -25,6 +26,7 @@ use apiserver::datastore::key::{Key, KeyType};
 use apiserver::datastore::serialization::{to_pairs, to_pairs_with_prefix};
 use apiserver::datastore::{self, DataStore, FilesystemDataStore, ScalarError};
 use apiserver::model;
+use apiserver::modeled_types::SingleLineString;
 use data_store_version::Version;
 
 // FIXME Get these from configuration in the future
@@ -36,6 +38,7 @@ mod error {
 
  use apiserver::datastore::key::KeyType;
  use apiserver::datastore::{self, serialization, ScalarError};
+ use apiserver::modeled_types::error::Error as ModeledTypesError;
  use data_store_version::error::Error as DataStoreVersionError;
  use snafu::Snafu;
 
@@ -67,6 +70,9 @@ mod error {
  #[snafu(display("defaults.toml's metadata is not a TOML list of Metadata"))]
  DefaultsMetadataNotTable { source: toml::de::Error },
 
+ #[snafu(display("defaults.toml's metadata has unexpected types"))]
+ DefaultsMetadataUnexpectedFormat {},
+
  #[snafu(display("Error querying datstore for populated keys: {}", source))]
  QueryData { source: datastore::Error },
 
@@ -103,6 +109,12 @@ mod error {
 
  #[snafu(display("Logger setup error: {}", source))]
  Logger { source: simplelog::TermLogError },
+
+ #[snafu(display("Internal error: {}", msg))]
+ Internal { msg: String },
+
+ #[snafu(display("Keys can't contain newlines: {}", source))]
+ SingleLineString { source: ModeledTypesError },
  }
 }
 
@@ -169,6 +181,100 @@ fn create_new_datastore<P: AsRef<Path>>(base_path: P, version: Option<Version>)
  Ok(())
 }
 
+/// Convert the generic toml::Value representing metadata into a
+/// Vec<Metadata> that can be used to write the metadata to the datastore.
+// The input to this function is a toml::Value that represents the metadata
+// read from defaults.toml. This table is structured like so:
+//
+// Table({"settings": Table({"foo": Table({"affected-services": Array([ ... ])})})})
+//
+// This function will convert the above table to a Vec<model::Metadata>,
+// validating the types and structure. The resulting Vec looks like:
+//
+// [
+// Metadata {key: "settings.hostname", md: "affected-services", val: Array([ ... ])},
+// Metadata { ... },
+// ]
+fn parse_metadata_toml(md_toml_val: toml::Value) -> Result<Vec<model::Metadata>> {
+ debug!("Parsing metadata toml");
+ let mut def_metadatas: Vec<model::Metadata> = Vec::new();
+
+ // Do a breadth-first search of the toml::Value table.
+ // Create a Vec of tuples to keep track of where we have visited in the
+ // toml::Value data structure. The first value in the tuple is the key
+ // (represented as a Vec of key segments), the second is the toml::Value
+ // associated with that key. It ends up looking like:
+ // [
+ // (
+ // ["settings", "hostname"],
+ // toml::Value
+ // ),
+ // ...
+ // ]
+ // For each key/value of the table we visit, match on the value of the
+ // table. If it's another table, we add it to the list to process its
+ // contents. If it is an array or string, we can construct a
+ // model::Metadata, and add it to the Vec of model::Metadata to be
+ // returned from the function.
+
+ // Start at the root of the tree.
+ let mut to_process = vec![(Vec::new(), md_toml_val)];
+
+ while !to_process.is_empty() {
+ let (mut path, toml_value) = to_process.pop().unwrap();
+ trace!("Current metadata table path: {:#?}", &path);
+
+ match toml_value {
+ // A table means there is more processing to do. Add the current
+ // key and value to the Vec to be processed further.
+ toml::Value::Table(table) => {
+ for (key, val) in table {
+ trace!("Found table for key '{}'", &key);
+ let mut path = path.clone();
+ path.push(key.to_string());
+ to_process.push((path, val));
+ }
+ }
+
+ // An array or string means we're ready to create a model::Metadata
+ val @ toml::Value::Array(_) | val @ toml::Value::String(_) => {
+ // Get the metadata key from the end of the path
+ let md_key = path.pop().context(error::Internal {
+ msg: "parse_metadata_toml found empty 'path' in the to_process vec - is 'metadata' not a Table?",
+ })?;
+
+ // Make sure that the path contains more than 1 item, i.e. ["settings", "hostname"]
+ ensure!(
+ path.len() >= 1,
+ error::Internal {
+ msg: format!(
+ "Cannot create empty metadata data key - is root not a Table?"
+ )
+ }
+ );
+ let data_key = path.join(".");
+
+ trace!(
+ "Found metadata key '{}' for data key '{}'",
+ &md_key,
+ &data_key
+ );
+
+ // Ensure the metadata/data keys don't contain newline chars
+ let md = SingleLineString::try_from(md_key).context(error::SingleLineString)?;
+ let key = SingleLineString::try_from(data_key).context(error::SingleLineString)?;
+
+ // Create the Metadata struct
+ def_metadatas.push(model::Metadata { key, md, val })
+ }
+
+ // We don't recognize any other values yet, something is awry
+ _ => return error::DefaultsMetadataUnexpectedFormat {}.fail(),
+ };
+ }
+ Ok(def_metadatas)
+}
+
 /// Creates a new FilesystemDataStore at the given path, with data and metadata coming from
 /// defaults.toml at compile time.
 fn populate_default_datastore<P: AsRef<Path>>(
@@ -262,9 +368,8 @@ fn populate_default_datastore<P: AsRef<Path>>(
  // If we have metadata, write it out to the datastore in Live state
  if let Some(def_metadata_val) = maybe_metadata_val {
  debug!("Serializing metadata and writing new keys to datastore");
- let def_metadatas: Vec<model::Metadata> = def_metadata_val
- .try_into()
- .context(error::DefaultsMetadataNotTable)?;
+ // Create a Vec<Metadata> from the metadata toml::Value
+ let def_metadatas = parse_metadata_toml(def_metadata_val)?;
 
  // Before this transformation, `existing_metadata` is a
  // map of data key to set of metadata keys: