updog and migrator: signed migrations

Author: webern

packages/os/migrator.service

@@ -3,7 +3,7 @@ Description=Bottlerocket data store migrator
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/migrator --datastore-path /var/lib/bottlerocket/datastore/current --migration-directories /var/lib/bottlerocket-migrations --migrate-to-version-from-os-release
+ExecStart=/usr/bin/migrator --datastore-path /var/lib/bottlerocket/datastore/current --migration-directory /var/lib/bottlerocket-migrations --root-path /usr/share/updog/root.json --metadata-directory /var/cache/bottlerocket-metadata --migrate-to-version-from-os-release
 RemainAfterExit=true
 StandardError=journal+console
 

sources/Cargo.lock

@@ -11,13 +11,18 @@ build = "build.rs"
 bottlerocket-release = { path = "../../../bottlerocket-release" }
 lazy_static = "1.2"
 log = "0.4"
+lz4 = "1.23.1"
 nix = "0.17"
+pentacle = "0.1.1"
 rand = { version = "0.7", default-features = false, features = ["std"] }
 regex = "1.1"
 semver = "0.9"
 simplelog = "0.7"
 snafu = "0.6"
+tempfile = "3.1.0"
+tough = "0.6.0"
 update_metadata = { path = "../../../updater/update_metadata" }
+url = "2.1.1"
 
 [build-dependencies]
 cargo-readme = "3.1"

sources/api/migration/migrator/Cargo.toml

@@ -15,7 +15,9 @@ fn usage() -> ! {
     eprintln!(
         r"Usage: {}
             --datastore-path PATH
-            --migration-directories PATH[:PATH:PATH...]
+            --migration-directory PATH
+            --root-path PATH
+            --metadata-directory PATH
             (--migrate-to-version x.y | --migrate-to-version-from-os-release)
             [ --no-color ]
             [ --log-level trace|debug|info|warn|error ]",
@@ -34,8 +36,10 @@ fn usage_msg<S: AsRef<str>>(msg: S) -> ! {
 pub(crate) struct Args {
     pub(crate) datastore_path: PathBuf,
     pub(crate) log_level: LevelFilter,
-    pub(crate) migration_directories: Vec<PathBuf>,
+    pub(crate) migration_directory: PathBuf,
     pub(crate) migrate_to_version: Version,
+    pub(crate) root_path: PathBuf,
+    pub(crate) metadata_directory: PathBuf,
 }
 
 impl Args {
@@ -44,8 +48,10 @@ impl Args {
         // Required parameters.
         let mut datastore_path = None;
         let mut log_level = None;
-        let mut migration_directories = None;
+        let mut migration_directory = None;
         let mut migrate_to_version = None;
+        let mut root_path = None;
+        let mut metadata_path = None;
 
         let mut iter = args.skip(1);
         while let Some(arg) = iter.next() {
@@ -84,16 +90,12 @@ impl Args {
                     }));
                 }
 
-                "--migration-directories" => {
-                    let paths_str = iter.next().unwrap_or_else(|| {
-                        usage_msg("Did not give argument to --migration-directories")
+                "--migration-directory" => {
+                    let path_str = iter.next().unwrap_or_else(|| {
+                        usage_msg("Did not give argument to --migration-directory")
                     });
-                    trace!("Given --migration-directories: {}", paths_str);
-                    let paths: Vec<_> = paths_str.split(':').map(PathBuf::from).collect();
-                    if paths.is_empty() {
-                        usage_msg("Found no paths in argument to --migration-directories");
-                    }
-                    migration_directories = Some(paths);
+                    trace!("Given --migration-directory: {}", path_str);
+                    migration_directory = Some(PathBuf::from(path_str));
                 }
 
                 "--migrate-to-version" => {
@@ -114,15 +116,40 @@ impl Args {
                     migrate_to_version = Some(br.version_id)
                 }
 
-                _ => usage(),
+                "--root-path" => {
+                    let path_str = iter
+                        .next()
+                        .unwrap_or_else(|| usage_msg("Did not give argument to --root-path"));
+                    trace!("Given --root-path: {}", path_str);
+                    root_path = Some(PathBuf::from(path_str));
+                }
+
+                "--metadata-directory" => {
+                    let path_str = iter.next().unwrap_or_else(|| {
+                        usage_msg("Did not give argument to --metadata-directory")
+                    });
+                    trace!("Given --metadata-directory: {}", path_str);
+                    metadata_path = Some(PathBuf::from(path_str));
+                }
+                _ => usage_msg(format!("Unable to parse input '{}'", arg)),
             }
         }
 
         Self {
-            datastore_path: datastore_path.unwrap_or_else(|| usage()),
+            datastore_path: datastore_path
+                .unwrap_or_else(|| usage_msg("--datastore-path must be specified")),
             log_level: log_level.unwrap_or_else(|| LevelFilter::Info),
-            migration_directories: migration_directories.unwrap_or_else(|| usage()),
-            migrate_to_version: migrate_to_version.unwrap_or_else(|| usage()),
+            migration_directory: migration_directory
+                .unwrap_or_else(|| usage_msg("--migration-directory must be specified")),
+            migrate_to_version: migrate_to_version.unwrap_or_else(|| {
+                usage_msg(
+                    "Desired version could not be determined; pass --migrate-to-version or \
+                    --migrate-to-version-from-os-release",
+                )
+            }),
+            root_path: root_path.unwrap_or_else(|| usage_msg("--root-path must be specified")),
+            metadata_directory: metadata_path
+                .unwrap_or_else(|| usage_msg("--metadata-directory must be specified")),
         }
     }
 }

sources/api/migration/migrator/src/args.rs

@@ -2,7 +2,7 @@
 //! is moving forward to a new version or rolling back to a previous version.
 
 use semver::Version;
-use std::cmp::Ordering;
+use std::cmp::{Ord, Ordering};
 use std::fmt;
 
 /// Direction represents whether we're moving forward toward a newer version, or rolling back to
@@ -45,14 +45,20 @@ mod test {
         let v02 = Version::new(0, 0, 2);
         let v10 = Version::new(0, 1, 0);
 
-        assert_eq!(Direction::from_versions(&v01, &v02), Some(Direction::Forward));
+        assert_eq!(
+            Direction::from_versions(&v01, &v02),
+            Some(Direction::Forward)
+        );
         assert_eq!(
             Direction::from_versions(&v02, &v01),
             Some(Direction::Backward)
         );
         assert_eq!(Direction::from_versions(&v01, &v01), None);
 
-        assert_eq!(Direction::from_versions(&v02, &v10), Some(Direction::Forward));
+        assert_eq!(
+            Direction::from_versions(&v02, &v10),
+            Some(Direction::Forward)
+        );
         assert_eq!(
             Direction::from_versions(&v10, &v02),
             Some(Direction::Backward)

sources/api/migration/migrator/src/direction.rs

@@ -4,7 +4,7 @@ use semver::Version;
 use snafu::Snafu;
 use std::io;
 use std::path::PathBuf;
-use std::process::{Command, Output};
+use std::process::Output;
 
 /// Error contains the errors that can happen during migration.
 #[derive(Debug, Snafu)]
@@ -13,6 +13,9 @@ pub(crate) enum Error {
     #[snafu(display("Internal error: {}", msg))]
     Internal { msg: String },
 
+    #[snafu(display("Unable to create tempdir for tough datastore: '{}'", source))]
+    CreateToughTempDir { source: std::io::Error },
+
     #[snafu(display("Data store path '{}' contains invalid UTF-8", path.display()))]
     DataStorePathNotUTF8 { path: PathBuf },
 
@@ -22,6 +25,14 @@ pub(crate) enum Error {
     #[snafu(display("Data store link '{}' points to /", path.display()))]
     DataStoreLinkToRoot { path: PathBuf },
 
+    #[snafu(display("Unable to create URL from path '{}'", path.display()))]
+    DirectoryUrl { path: PathBuf },
+
+    #[snafu(display("Error finding migration: {}", source))]
+    FindMigrations {
+        source: update_metadata::error::Error,
+    },
+
     #[snafu(display("Data store path '{}' contains invalid version: {}", path.display(), source))]
     InvalidDataStoreVersion {
         path: PathBuf,
@@ -37,8 +48,11 @@ pub(crate) enum Error {
     #[snafu(display("Data store for new version {} already exists at {}", version, path.display()))]
     NewVersionAlreadyExists { version: Version, path: PathBuf },
 
-    #[snafu(display("Unable to start migration command {:?} - {}", command, source))]
-    StartMigration { command: Command, source: io::Error },
+    #[snafu(display("Unable to seal migration command: {}", source))]
+    SealMigration { source: std::io::Error },
+
+    #[snafu(display("Unable to start migration command: {}", source))]
+    StartMigration { source: std::io::Error },
 
     #[snafu(display("Migration returned '{}' - stderr: {}",
                     output.status.code()
@@ -59,9 +73,38 @@ pub(crate) enum Error {
     #[snafu(display("Failed listing migration directory '{}': {}", dir.display(), source))]
     ListMigrations { dir: PathBuf, source: io::Error },
 
+    #[snafu(display("Error loading manifest: {}", source))]
+    LoadManifest {
+        source: update_metadata::error::Error,
+    },
+
+    #[snafu(display("Error loading migration '{}': {}", migration, source))]
+    LoadMigration {
+        migration: String,
+        source: tough::error::Error,
+    },
+
+    #[snafu(display("Failed to decode LZ4-compressed migration {}: {}", migration, source))]
+    Lz4Decode {
+        migration: String,
+        source: std::io::Error,
+    },
+
+    #[snafu(display("Migration '{}' not found", migration))]
+    MigrationNotFound { migration: String },
+
+    #[snafu(display("Failed to open trusted root metadata file {}: {}", path.display(), source))]
+    OpenRoot {
+        path: PathBuf,
+        source: std::io::Error,
+    },
+
     #[snafu(display("Failed reading migration directory entry: {}", source))]
     ReadMigrationEntry { source: io::Error },
 
+    #[snafu(display("Failed to load TUF repo: {}", source))]
+    RepoLoad { source: tough::error::Error },
+
     #[snafu(display("Failed reading metadata of '{}': {}", path.display(), source))]
     PathMetadata { path: PathBuf, source: io::Error },
 
@@ -70,9 +113,6 @@ pub(crate) enum Error {
 
     #[snafu(display("Migration path '{}' contains invalid UTF-8", path.display()))]
     MigrationNameNotUTF8 { path: PathBuf },
-
-    #[snafu(display("Logger setup error: {}", source))]
-    Logger { source: simplelog::TermLogError },
 }
 
 /// Result alias containing our Error type.