migrator: remove all the weak setting and settings-generator

We will remove the weak settings and settings genrators using the
migrator of the destination migrator.
The storewolf do not repopulate any metadata or setting, if it is
already presnt. As migrator runs before storewolf, if we will delete the
weak settings and setting-generators in migrator, storewolf can populate
the setting-generator from defaults and ssundog will populate the new
source using the new setting generator from default.

sources/api/migration/migrator/Cargo.toml

@@ -20,17 +20,22 @@ futures = { workspace = true, features = ["default"] }
 futures-core.workspace = true
 log.workspace = true
 lz4.workspace = true
+models.workspace = true
 nix.workspace = true
 pentacle.workspace = true
 rand = { workspace = true, features = ["std", "std_rng"] }
 semver.workspace = true
+serde_json.workspace = true
 simplelog.workspace = true
 snafu.workspace = true
 tokio = { workspace = true, features = ["fs", "macros", "rt-multi-thread"] }
 tokio-util = { workspace = true, features = ["compat", "io-util"] }
 tough = { workspace = true }
 update_metadata.workspace = true
 url.workspace = true
+walkdir.workspace = true
+datastore.workspace = true
+percent-encoding.workspace = true
 
 [build-dependencies]
 generate-readme.workspace = true

sources/api/migration/migrator/src/datastore_helper.rs

@@ -0,0 +1,116 @@
+//! This module contains the functions that interact with the data store, retrieving data to
+//! update and writing back updated data.
+
+use snafu::ResultExt;
+use std::collections::HashMap;
+
+use crate::{error, Result};
+use datastore::{deserialize_scalar, serialize_scalar, Committed, DataStore, Key, KeyType, Value};
+
+/// Mapping of metadata key name to arbitrary value.  Each data key can have a Metadata describing
+/// its metadata keys.
+/// example: Key: settings.host-containers.admin.source, Metadata: strength and Value: "weak"
+pub type Metadata = HashMap<String, Value>;
+
+/// DataStoreData holds all data that exists in datastore.
+// We will take this as an input and use it to delete weak settings and settings generator.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DataStoreData {
+    /// Mapping of data key names to their arbitrary values.
+    pub data: HashMap<String, Value>,
+    /// Mapping of data key names to their metadata.
+    pub metadata: HashMap<String, Metadata>,
+}
+
+// To get input data from the existing data store, we use datastore methods.
+// This method is private to the crate, so we can
+// reconsider as needed.
+/// Retrieves data from the specified data store in a consistent format for easy modification.
+pub(crate) fn get_input_data<D: DataStore>(
+    datastore: &D,
+    committed: &Committed,
+) -> Result<DataStoreData> {
+    let raw_data = datastore
+        .get_prefix("", committed)
+        .with_context(|_| error::GetDataSnafu {
+            committed: committed.clone(),
+        })?;
+
+    let mut data = HashMap::new();
+    for (data_key, value_str) in raw_data.into_iter() {
+        // Store keys with just their name, rather than the full Key.
+        let key_name = data_key.name();
+        // Deserialize values to Value so there's a consistent input type.  (We can't specify item
+        // types because we'd have to know the model structure.)
+        let value =
+            deserialize_scalar(&value_str).context(error::DeserializeSnafu { input: value_str })?;
+        data.insert(key_name.clone(), value);
+    }
+
+    let mut metadata = HashMap::new();
+
+    let raw_metadata = datastore
+        .get_metadata_prefix("", committed, &None as &Option<&str>)
+        .context(error::GetMetadataSnafu)?;
+    for (data_key, meta_map) in raw_metadata.into_iter() {
+        let data_key_name = data_key.name();
+        let data_entry = metadata
+            .entry(data_key_name.clone())
+            .or_insert_with(HashMap::new);
+        for (metadata_key, value_str) in meta_map.into_iter() {
+            let metadata_key_name = metadata_key.name();
+            let value = deserialize_scalar(&value_str)
+                .context(error::DeserializeSnafu { input: value_str })?;
+            data_entry.insert(metadata_key_name.clone(), value);
+        }
+    }
+    Ok(DataStoreData { data, metadata })
+}
+
+// Similar to get_input_data, we use datastore methods here; 
+// This method is also private to the crate, so we can reconsider as needed.
+/// Updates the given data store with the given (updated) data.
+pub(crate) fn set_output_data<D: DataStore>(
+    datastore: &mut D,
+    input: &DataStoreData,
+    committed: &Committed,
+) -> Result<()> {
+    // Prepare serialized data
+    let mut data = HashMap::new();
+    for (data_key_name, raw_value) in &input.data {
+        let data_key = Key::new(KeyType::Data, data_key_name).context(error::InvalidKeySnafu {
+            key_type: KeyType::Data,
+            key: data_key_name,
+        })?;
+        let value = serialize_scalar(raw_value).context(error::SerializeSnafu)?;
+        data.insert(data_key, value);
+    }
+
+    // This is one of the rare cases where we want to set keys directly in the datastore:
+    // * We're operating on a temporary copy of the datastore, so no concurrency issues
+    // * We're either about to reboot or just have, and the settings applier will run afterward
+    datastore
+        .set_keys(&data, committed)
+        .context(error::DataStoreWriteSnafu)?;
+
+    // Set metadata in a loop (currently no batch API)
+    for (data_key_name, meta_map) in &input.metadata {
+        let data_key = Key::new(KeyType::Data, data_key_name).context(error::InvalidKeySnafu {
+            key_type: KeyType::Data,
+            key: data_key_name,
+        })?;
+        for (metadata_key_name, raw_value) in meta_map.iter() {
+            let metadata_key =
+                Key::new(KeyType::Meta, metadata_key_name).context(error::InvalidKeySnafu {
+                    key_type: KeyType::Meta,
+                    key: metadata_key_name,
+                })?;
+            let value = serialize_scalar(&raw_value).context(error::SerializeSnafu)?;
+            datastore
+                .set_metadata(&metadata_key, &data_key, value, committed)
+                .context(error::DataStoreWriteSnafu)?;
+        }
+    }
+
+    Ok(())
+}

sources/api/migration/migrator/src/error.rs

@@ -106,6 +106,51 @@ pub(crate) enum Error {
         #[snafu(source(from(tough::error::Error, Box::new)))]
         source: Box<tough::error::Error>,
     },
+
+    #[snafu(display("Unable to get {:?} data from datastore: {}", committed, source))]
+    GetData {
+        committed: datastore::Committed,
+        #[snafu(source(from(datastore::Error, Box::new)))]
+        source: Box<datastore::Error>,
+    },
+
+    #[snafu(display("Unable to deserialize to Value from '{}': {}", input, source))]
+    Deserialize {
+        input: String,
+        source: datastore::ScalarError,
+    },
+
+    #[snafu(display("Unable to get metadata from datastore: {}", source))]
+    GetMetadata {
+        #[snafu(source(from(datastore::Error, Box::new)))]
+        source: Box<datastore::Error>,
+    },
+
+    #[snafu(display("Update used invalid {:?} key '{}': {}", key_type, key, source))]
+    InvalidKey {
+        key_type: datastore::KeyType,
+        key: String,
+        #[snafu(source(from(datastore::Error, Box::new)))]
+        source: Box<datastore::Error>,
+    },
+
+    #[snafu(display("Unable to write to data store: {}", source))]
+    DataStoreWrite {
+        #[snafu(source(from(datastore::Error, Box::new)))]
+        source: Box<datastore::Error>,
+    },
+
+    #[snafu(display("Unable to serialize Value: {}", source))]
+    Serialize { source: datastore::ScalarError },
+
+    #[snafu(display("Unable to list transactions in data store: {}", source))]
+    ListTransactions {
+        #[snafu(source(from(datastore::Error, Box::new)))]
+        source: Box<datastore::Error>,
+    },
+
+    #[snafu(display("Error deserializing response value to SettingsGenerator: {}", source))]
+    DeserializeSettingsGenerator { source: serde_json::Error },
 }
 
 /// Result alias containing our Error type.

sources/api/migration/migrator/src/main.rs

@@ -22,14 +22,18 @@
 extern crate log;
 
 use args::Args;
+use datastore::{Committed, DataStore, FilesystemDataStore, Value};
+use datastore_helper::{get_input_data, set_output_data, DataStoreData};
 use direction::Direction;
 use error::Result;
 use futures::{StreamExt, TryStreamExt};
+use model::SettingsGenerator;
 use nix::{dir::Dir, fcntl::OFlag, sys::stat::Mode, unistd::fsync};
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 use semver::Version;
 use simplelog::{Config as LogConfig, SimpleLogger};
 use snafu::{ensure, OptionExt, ResultExt};
+use std::collections::HashSet;
 use std::convert::TryInto;
 use std::env;
 use std::io::ErrorKind;
@@ -46,11 +50,14 @@ use update_metadata::Manifest;
 use url::Url;
 
 mod args;
+mod datastore_helper;
 mod direction;
 mod error;
 #[cfg(test)]
 mod test;
 
+type DataStoreImplementation = FilesystemDataStore;
+
 // Returning a Result from main makes it print a Debug representation of the error, but with Snafu
 // we have nice Display representations of the error, so we wrap "main" (run) and print any error.
 // https://github.com/shepmaster/snafu/issues/110
@@ -178,18 +185,20 @@ pub(crate) async fn run(args: &Args) -> Result<()> {
         update_metadata::find_migrations(&current_version, &args.migrate_to_version, &manifest)
             .context(error::FindMigrationsSnafu)?;
 
+    let datastore = remove_weak_settings(&args.datastore_path, &args.migrate_to_version).await?;
+
     if migrations.is_empty() {
         // Not all new OS versions need to change the data store format.  If there's been no
         // change, we can just link to the last version rather than making a copy.
         // (Note: we link to the fully resolved directory, args.datastore_path,  so we don't
         // have a chain of symlinks that could go past the maximum depth.)
-        flip_to_new_version(&args.migrate_to_version, &args.datastore_path).await?;
+        flip_to_new_version(&args.migrate_to_version, &datastore).await?;
     } else {
         let copy_path = run_migrations(
             &repo,
             direction,
             &migrations,
-            &args.datastore_path,
+            &datastore,
             &args.migrate_to_version,
         )
         .await?;
@@ -234,6 +243,79 @@ where
     Ok(to)
 }
 
+async fn remove_weak_settings<P>(datastore_path: P, new_version: &Version) -> Result<PathBuf>
+where
+    P: AsRef<Path>,
+{
+    // We start with the given source_datastore, updating this to delete weak settings and setting-generators
+    let source_datastore = datastore_path.as_ref();
+    // We create a new data store (below) to serve as the target for update.  (Start at
+    // source just to have the right type)
+    let target_datastore = new_datastore_location(source_datastore, new_version)?;
+
+    let source = DataStoreImplementation::new(source_datastore);
+    let mut target = DataStoreImplementation::new(&target_datastore);
+
+    // Run for both live data and pending transactions
+    let mut committeds = vec![Committed::Live];
+    let transactions = source
+        .list_transactions()
+        .context(error::ListTransactionsSnafu)?;
+    committeds.extend(transactions.into_iter().map(|tx| Committed::Pending { tx }));
+
+    for committed in committeds {
+        let input = get_input_data(&source, &committed)?;
+
+        let mut migrated = input.clone();
+        let input_after_removing_weak_settings = remove_weak_setting_from_datastore(&mut migrated)?;
+
+        set_output_data(&mut target, &input_after_removing_weak_settings, &committed)?;
+    }
+
+    Ok(target_datastore)
+}
+
+fn remove_weak_setting_from_datastore(datastore: &mut DataStoreData) -> Result<DataStoreData> {
+    let mut keys_to_remove = HashSet::new();
+
+    // Collect the metadata keys whose strength is weak 
+    for (key, inner_map) in &datastore.metadata {
+        if let Some(strength) = inner_map.get("strength") {
+            if strength == &Value::String("weak".to_string()) {
+                keys_to_remove.insert(key.clone());
+            }
+        }
+    }
+    // Remove strength metadata for weak settings and weak settings
+    for key in keys_to_remove {
+        let metadata = datastore.metadata.get(&key);
+        if let Some(metadata) = metadata {
+            let mut inner_map = metadata.clone();
+            inner_map.remove("strength");
+            datastore.metadata.insert(key.clone(), inner_map);
+        }
+        datastore.data.remove(&key);
+    }
+
+    // Remove all the setting generators with weak strength
+    // We just need to delete all the weak settings-genrator
+    // as this datastore will be input for migrations and those depends on other metadata in datastore.
+    for (key, inner_map) in datastore.metadata.clone() {
+        if let Some(Value::Object(obj)) = inner_map.get("setting-generator") {
+            let setting_generator: SettingsGenerator =
+                serde_json::from_value(Value::Object(obj.clone()))
+                    .context(error::DeserializeSettingsGeneratorSnafu)?;
+            if setting_generator.is_weak() {
+                let mut inner_map = inner_map.clone();
+                inner_map.remove("setting-generator");
+                datastore.metadata.insert(key.clone(), inner_map);
+            }
+        }
+    }
+
+    Ok(datastore.clone())
+}
+
 /// Runs the given migrations in their given order.  The given direction is passed to each
 /// migration so it knows which direction we're migrating.
 ///

sources/api/migration/migrator/src/test.rs

@@ -532,3 +532,4 @@ async fn migrate_backward_with_failed_migration() {
         .unwrap()
         .starts_with("v0.99.1"));
 }
+