Merge pull request #5117 from cgwalters/edition

treefile: Add an `edition` && treefile: Add finalize.d

docs/treefile.md

@@ -14,6 +14,10 @@ Jenkins to operate on them as it changes.
 
 It supports the following parameters:
 
+ * `edition`: string, optional: If not set, the default value is
+   treated as `2014`. The only other supported value is `2024`, which
+   changes some defaults.
+
  * `ref`: string, mandatory: Holds a string which will be the name of
    the branch for the content. This field supports variable substitution.
 
@@ -47,7 +51,8 @@ It supports the following parameters:
       upgrading from very old versions of libostree.
     * "modules": Kernel data goes just in `/usr/lib/modules`.  Use
       this for new systems, and systems that don't need to be upgraded
-      from very old libostree versions.
+      from very old libostree versions. This is the default for editions 2024
+      and above.
 
  * `etc-group-members`: Array of strings, optional: Unix groups in this
    list will be stored in `/etc/group` instead of `/usr/lib/group`.  Use
@@ -441,8 +446,8 @@ It supports the following parameters:
    supported. For more details, see the OSTree manual:
    https://ostreedev.github.io/ostree/deployment/
 
- * `tmp-is-dir`: boolean, optional: Defaults to `false`.  By default,
-   rpm-ostree creates symlink `/tmp` → `sysroot/tmp`.  When set to `true`,
+ * `tmp-is-dir`: boolean, optional: Defaults to `false` in editions < 2024, otherwise `true`.
+   By default, rpm-ostree creates symlink `/tmp` → `sysroot/tmp`.  When set to `true`,
    `/tmp` will be a regular directory, which allows the `systemd` unit
    `tmp.mount` to mount it as `tmpfs`. It's more flexible to leave it
    as a directory, and further, we don't want to encourage `/sysroot`
@@ -521,3 +526,15 @@ version of `rpm-ostree`.
      and are purely machine-local state.
    - `root`: These are plain directories; only use this with composefs enabled!
 
+### Associated directories
+
+In edition `2024`, "associated directories" have been introduced as an experimental feature. These
+are "drop-in" style directories which can contain inline content or
+scripts. When processing a manifest file, if these subdirectories exist
+in the same directory as the manifest, they will be automatically used:
+
+- `finalize.d`: Executed synchronously in alphanumeric order from the
+  host/ambient environment (*not* from the target); the current working directory will be
+  the target root filesystem. There is no additional sandboxing or containerization
+  applied to the execution of the binary. The builtin "change detection"
+  is not applied to the content of the scripts.

rust/src/composepost.rs

@@ -1199,10 +1199,13 @@ fn workaround_selinux_cross_labeling_recurse(
 }
 
 /// This is the nearly the last code executed before we run `ostree commit`.
-pub fn compose_postprocess_final(rootfs_dfd: i32, _treefile: &Treefile) -> CxxResult<()> {
+pub fn compose_postprocess_final(rootfs_dfd: i32, treefile: &Treefile) -> CxxResult<()> {
     let rootfs = unsafe { &crate::ffiutil::ffi_dirfd(rootfs_dfd)? };
 
     hardlink_rpmdb_base_location(rootfs, None)?;
+
+    treefile.exec_finalize_d(rootfs)?;
+
     Ok(())
 }
 

rust/src/treefile.rs

@@ -22,7 +22,9 @@
 use crate::cxxrsutil::*;
 use anyhow::{anyhow, bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
+use cap_std::fs::MetadataExt as _;
 use cap_std_ext::cap_std::fs::Dir;
+use cap_std_ext::cmdext::CapStdExtCommandExt;
 use cap_std_ext::prelude::CapStdExtDirExt;
 use nix::unistd::{Gid, Uid};
 use once_cell::sync::Lazy;
@@ -36,6 +38,7 @@ use std::io::prelude::*;
 use std::os::unix::fs::{MetadataExt, PermissionsExt};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::path::{Path, PathBuf};
+use std::process::Command;
 use std::str::FromStr;
 use std::{fs, io};
 use tracing::{event, instrument, Level};
@@ -48,6 +51,9 @@ use crate::utils::OptionExtGetOrInsertDefault;
 
 const INCLUDE_MAXDEPTH: u32 = 50;
 
+// The directory with executable scripts for image finalization
+const FINALIZE_D: &str = "finalize.d";
+
 /// Path to the flattened JSON serialization of the treefile, installed on the target (client)
 /// filesystem.  Nothing actually parses this by default client side today,
 /// it's intended to be informative.
@@ -64,6 +70,7 @@ pub(crate) struct TreefileExternals {
     add_files: BTreeMap<String, fs::File>,
     passwd: Option<fs::File>,
     group: Option<fs::File>,
+    pub(crate) finalize_d: BTreeMap<String, Utf8PathBuf>,
 }
 
 // This type name is exposed through ffi.
@@ -160,6 +167,12 @@ fn treefile_parse_stream<R: io::Read>(
         treefile.check_groups = Some(CheckGroups::None);
     }
 
+    // Change these defaults for 2024 edition
+    if treefile.edition.unwrap_or_default() >= Edition::Twenty24 {
+        treefile.boot_location = Some(BootLocation::Modules);
+        treefile.tmp_is_dir = Some(true);
+    }
+
     // Special handling for packages, since we allow whitespace within items.
     // We also canonicalize bootstrap_packages to packages here so it's
     // easier to append the basearch packages after.
@@ -298,6 +311,30 @@ fn treefile_parse<P: AsRef<Path>>(
         }
     }
     let parent = utils::parent_dir(filename).unwrap();
+    let parent: &Utf8Path = parent.try_into()?;
+    let dir = Dir::open_ambient_dir(parent, cap_std::ambient_authority())?;
+    let finalize_d = if let Some(d) = dir.open_dir_optional(FINALIZE_D)? {
+        let mut r = BTreeMap::new();
+        for ent in d.entries()? {
+            let ent = ent?;
+            let meta = ent.metadata()?;
+            if !meta.is_file() {
+                continue;
+            }
+            if meta.mode() & libc::S_IXUSR == 0 {
+                continue;
+            }
+            let name = ent.file_name();
+            let name = name
+                .to_str()
+                .ok_or_else(|| anyhow::anyhow!("non UTF-8 name '{name:?}'"))?;
+            let path = format!("{parent}/{FINALIZE_D}/{name}");
+            r.insert(name.to_owned(), path.into());
+        }
+        r
+    } else {
+        BTreeMap::new()
+    };
     let passwd = match tf.get_check_passwd() {
         CheckPasswd::File(ref f) => load_passwd_file(&parent, f)?,
         _ => None,
@@ -314,6 +351,7 @@ fn treefile_parse<P: AsRef<Path>>(
             add_files,
             passwd,
             group,
+            finalize_d,
         },
     })
 }
@@ -389,6 +427,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
     }
 
     merge_basics!(
+        edition,
         treeref,
         basearch,
         rojig,
@@ -501,6 +540,10 @@ fn treefile_merge_externals(dest: &mut TreefileExternals, src: &mut TreefileExte
     if dest.group.is_none() {
         dest.group = src.group.take();
     }
+
+    while let Some((name, f)) = src.finalize_d.pop_first() {
+        dest.finalize_d.insert(name, f);
+    }
 }
 
 /// Recursively parse a treefile, merging along the way.
@@ -766,6 +809,21 @@ impl Treefile {
         Ok(rpackages)
     }
 
+    /// Execute all finalize.d scripts
+    pub(crate) fn exec_finalize_d(&self, rootfs: &Dir) -> Result<()> {
+        for (name, path) in self.externals.finalize_d.iter() {
+            println!("Executing: {name}");
+            let st = Command::new(path)
+                .cwd_dir(rootfs.try_clone()?)
+                .status()
+                .with_context(|| format!("exec {path:?}"))?;
+            if !st.success() {
+                anyhow::bail!("finalize.d {name} failed: {st:?}");
+            }
+        }
+        Ok(())
+    }
+
     pub(crate) fn add_packages(
         &mut self,
         packages: Vec<String>,
@@ -2391,10 +2449,22 @@ impl std::ops::DerefMut for TreeComposeConfig {
     }
 }
 
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq, Copy, Clone, PartialOrd, Ord)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) enum Edition {
+    #[serde(rename = "2014")]
+    #[default]
+    Twenty14,
+    #[serde(rename = "2024")]
+    Twenty24,
+}
+
 /// These fields are only useful when composing a new ostree commit.
 #[derive(Clone, Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
 #[serde(rename_all = "kebab-case")]
 pub(crate) struct BaseComposeConfigFields {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) edition: Option<Edition>,
     // Compose controls
     #[serde(rename = "ref")]
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -3792,6 +3862,62 @@ conditional-include:
         assert!(tf.parsed.base.tmp_is_dir.unwrap());
     }
 
+    #[test]
+    fn test_edition() {
+        let workdir = tempfile::tempdir().unwrap();
+        let workdir: &Utf8Path = workdir.path().try_into().unwrap();
+        let tf = indoc! { r#"
+            edition: "2024"
+        "#};
+        let tf = new_test_treefile(workdir, tf, None).unwrap();
+        assert!(tf.parsed.base.tmp_is_dir.unwrap());
+        assert!(tf.externals.finalize_d.is_empty());
+    }
+
+    #[test]
+    fn test_finalize() -> Result<()> {
+        let workdir = tempfile::tempdir().unwrap();
+        let workdir: &Utf8Path = workdir.path().try_into().unwrap();
+        let tf = indoc! { r#"
+            edition: "2024"
+        "#};
+        let finalize_d = workdir.join(FINALIZE_D);
+        std::fs::create_dir(&finalize_d).unwrap();
+        std::fs::write(
+            finalize_d.join("01-foo"),
+            indoc::indoc! { r#"
+            #!/bin/bash
+            touch foo
+            sleep 1
+        "# },
+        )?;
+        std::fs::write(
+            finalize_d.join("02-bar"),
+            indoc::indoc! { r#"
+            #!/bin/bash
+            touch bar
+        "# },
+        )?;
+        for ent in std::fs::read_dir(&finalize_d).unwrap() {
+            let ent = ent?;
+            std::fs::set_permissions(ent.path(), fs::Permissions::from_mode(0o755))?;
+        }
+        let rootpath = workdir.join("rootfs");
+        std::fs::create_dir(&rootpath)?;
+        let rootpath = Dir::open_ambient_dir(&rootpath, cap_std::ambient_authority())?;
+
+        let tf = new_test_treefile(workdir, tf, None).unwrap();
+        assert_eq!(tf.externals.finalize_d.len(), 2);
+
+        tf.exec_finalize_d(&rootpath)?;
+
+        let foometa = rootpath.symlink_metadata("foo").unwrap();
+        let barmeta = rootpath.symlink_metadata("bar").unwrap();
+        assert!(foometa.modified().unwrap() < barmeta.modified().unwrap());
+
+        Ok(())
+    }
+
     #[test]
     fn test_check_passwd() {
         {