From 4a12c44c110a717404f4289161e40b6307825a3f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 Oct 2024 13:10:18 -0400 Subject: [PATCH 1/2] treefile: Add an `edition` This will allow us to clean up some defaults and change some semantics. In particular, I'd like to add an opinionated mechanism to copy files from git into the build. For now, `edition: "2024"` just flips on `tmp-is-dir: true` and fixes `boot-location`. Signed-off-by: Colin Walters --- docs/treefile.md | 11 ++++++++--- rust/src/treefile.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/treefile.md b/docs/treefile.md index de56f431d4..1c62b47639 100644 --- a/docs/treefile.md +++ b/docs/treefile.md @@ -14,6 +14,10 @@ Jenkins to operate on them as it changes. It supports the following parameters: + * `edition`: string, optional: If not set, the default value is + treated as `2014`. The only other supported value is `2024`, which + changes some defaults. + * `ref`: string, mandatory: Holds a string which will be the name of the branch for the content. This field supports variable substitution. @@ -47,7 +51,8 @@ It supports the following parameters: upgrading from very old versions of libostree. * "modules": Kernel data goes just in `/usr/lib/modules`. Use this for new systems, and systems that don't need to be upgraded - from very old libostree versions. + from very old libostree versions. This is the default for editions 2024 + and above. * `etc-group-members`: Array of strings, optional: Unix groups in this list will be stored in `/etc/group` instead of `/usr/lib/group`. Use @@ -441,8 +446,8 @@ It supports the following parameters: supported. For more details, see the OSTree manual: https://ostreedev.github.io/ostree/deployment/ - * `tmp-is-dir`: boolean, optional: Defaults to `false`. By default, - rpm-ostree creates symlink `/tmp` → `sysroot/tmp`. When set to `true`, + * `tmp-is-dir`: boolean, optional: Defaults to `false` in editions < 2024, otherwise `true`. + By default, rpm-ostree creates symlink `/tmp` → `sysroot/tmp`. When set to `true`, `/tmp` will be a regular directory, which allows the `systemd` unit `tmp.mount` to mount it as `tmpfs`. It's more flexible to leave it as a directory, and further, we don't want to encourage `/sysroot` diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 85a09a9683..929a2cbf21 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -160,6 +160,12 @@ fn treefile_parse_stream( treefile.check_groups = Some(CheckGroups::None); } + // Change these defaults for 2024 edition + if treefile.edition.unwrap_or_default() >= Edition::Twenty24 { + treefile.boot_location = Some(BootLocation::Modules); + treefile.tmp_is_dir = Some(true); + } + // Special handling for packages, since we allow whitespace within items. // We also canonicalize bootstrap_packages to packages here so it's // easier to append the basearch packages after. @@ -389,6 +395,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) { } merge_basics!( + edition, treeref, basearch, rojig, @@ -2391,10 +2398,22 @@ impl std::ops::DerefMut for TreeComposeConfig { } } +#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq, Copy, Clone, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Edition { + #[serde(rename = "2014")] + #[default] + Twenty14, + #[serde(rename = "2024")] + Twenty24, +} + /// These fields are only useful when composing a new ostree commit. #[derive(Clone, Serialize, Deserialize, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub(crate) struct BaseComposeConfigFields { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) edition: Option, // Compose controls #[serde(rename = "ref")] #[serde(skip_serializing_if = "Option::is_none")] @@ -3792,6 +3811,17 @@ conditional-include: assert!(tf.parsed.base.tmp_is_dir.unwrap()); } + #[test] + fn test_edition() { + let workdir = tempfile::tempdir().unwrap(); + let workdir: &Utf8Path = workdir.path().try_into().unwrap(); + let tf = indoc! { r#" + edition: "2024" + "#}; + let tf = new_test_treefile(workdir, tf, None).unwrap(); + assert!(tf.parsed.base.tmp_is_dir.unwrap()); + } + #[test] fn test_check_passwd() { { From bfabf576523c541356765818c6346f6b63586607 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 Oct 2024 20:22:11 -0400 Subject: [PATCH 2/2] treefile: Add finalize.d as experimental This is a total escape hatch for arbitrary mutation of the filesystem tree just before we do an ostree commit. Signed-off-by: Colin Walters --- docs/treefile.md | 12 ++++++ rust/src/composepost.rs | 5 ++- rust/src/treefile.rs | 96 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/docs/treefile.md b/docs/treefile.md index 1c62b47639..7226679cba 100644 --- a/docs/treefile.md +++ b/docs/treefile.md @@ -526,3 +526,15 @@ version of `rpm-ostree`. and are purely machine-local state. - `root`: These are plain directories; only use this with composefs enabled! +### Associated directories + +In edition `2024`, "associated directories" have been introduced as an experimental feature. These +are "drop-in" style directories which can contain inline content or +scripts. When processing a manifest file, if these subdirectories exist +in the same directory as the manifest, they will be automatically used: + +- `finalize.d`: Executed synchronously in alphanumeric order from the + host/ambient environment (*not* from the target); the current working directory will be + the target root filesystem. There is no additional sandboxing or containerization + applied to the execution of the binary. The builtin "change detection" + is not applied to the content of the scripts. diff --git a/rust/src/composepost.rs b/rust/src/composepost.rs index 1a5ae86937..72ca67ea61 100644 --- a/rust/src/composepost.rs +++ b/rust/src/composepost.rs @@ -1199,10 +1199,13 @@ fn workaround_selinux_cross_labeling_recurse( } /// This is the nearly the last code executed before we run `ostree commit`. -pub fn compose_postprocess_final(rootfs_dfd: i32, _treefile: &Treefile) -> CxxResult<()> { +pub fn compose_postprocess_final(rootfs_dfd: i32, treefile: &Treefile) -> CxxResult<()> { let rootfs = unsafe { &crate::ffiutil::ffi_dirfd(rootfs_dfd)? }; hardlink_rpmdb_base_location(rootfs, None)?; + + treefile.exec_finalize_d(rootfs)?; + Ok(()) } diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 929a2cbf21..b43f0fc75c 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -22,7 +22,9 @@ use crate::cxxrsutil::*; use anyhow::{anyhow, bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; +use cap_std::fs::MetadataExt as _; use cap_std_ext::cap_std::fs::Dir; +use cap_std_ext::cmdext::CapStdExtCommandExt; use cap_std_ext::prelude::CapStdExtDirExt; use nix::unistd::{Gid, Uid}; use once_cell::sync::Lazy; @@ -36,6 +38,7 @@ use std::io::prelude::*; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::os::unix::io::{AsRawFd, RawFd}; use std::path::{Path, PathBuf}; +use std::process::Command; use std::str::FromStr; use std::{fs, io}; use tracing::{event, instrument, Level}; @@ -48,6 +51,9 @@ use crate::utils::OptionExtGetOrInsertDefault; const INCLUDE_MAXDEPTH: u32 = 50; +// The directory with executable scripts for image finalization +const FINALIZE_D: &str = "finalize.d"; + /// Path to the flattened JSON serialization of the treefile, installed on the target (client) /// filesystem. Nothing actually parses this by default client side today, /// it's intended to be informative. @@ -64,6 +70,7 @@ pub(crate) struct TreefileExternals { add_files: BTreeMap, passwd: Option, group: Option, + pub(crate) finalize_d: BTreeMap, } // This type name is exposed through ffi. @@ -304,6 +311,30 @@ fn treefile_parse>( } } let parent = utils::parent_dir(filename).unwrap(); + let parent: &Utf8Path = parent.try_into()?; + let dir = Dir::open_ambient_dir(parent, cap_std::ambient_authority())?; + let finalize_d = if let Some(d) = dir.open_dir_optional(FINALIZE_D)? { + let mut r = BTreeMap::new(); + for ent in d.entries()? { + let ent = ent?; + let meta = ent.metadata()?; + if !meta.is_file() { + continue; + } + if meta.mode() & libc::S_IXUSR == 0 { + continue; + } + let name = ent.file_name(); + let name = name + .to_str() + .ok_or_else(|| anyhow::anyhow!("non UTF-8 name '{name:?}'"))?; + let path = format!("{parent}/{FINALIZE_D}/{name}"); + r.insert(name.to_owned(), path.into()); + } + r + } else { + BTreeMap::new() + }; let passwd = match tf.get_check_passwd() { CheckPasswd::File(ref f) => load_passwd_file(&parent, f)?, _ => None, @@ -320,6 +351,7 @@ fn treefile_parse>( add_files, passwd, group, + finalize_d, }, }) } @@ -508,6 +540,10 @@ fn treefile_merge_externals(dest: &mut TreefileExternals, src: &mut TreefileExte if dest.group.is_none() { dest.group = src.group.take(); } + + while let Some((name, f)) = src.finalize_d.pop_first() { + dest.finalize_d.insert(name, f); + } } /// Recursively parse a treefile, merging along the way. @@ -773,6 +809,21 @@ impl Treefile { Ok(rpackages) } + /// Execute all finalize.d scripts + pub(crate) fn exec_finalize_d(&self, rootfs: &Dir) -> Result<()> { + for (name, path) in self.externals.finalize_d.iter() { + println!("Executing: {name}"); + let st = Command::new(path) + .cwd_dir(rootfs.try_clone()?) + .status() + .with_context(|| format!("exec {path:?}"))?; + if !st.success() { + anyhow::bail!("finalize.d {name} failed: {st:?}"); + } + } + Ok(()) + } + pub(crate) fn add_packages( &mut self, packages: Vec, @@ -3820,6 +3871,51 @@ conditional-include: "#}; let tf = new_test_treefile(workdir, tf, None).unwrap(); assert!(tf.parsed.base.tmp_is_dir.unwrap()); + assert!(tf.externals.finalize_d.is_empty()); + } + + #[test] + fn test_finalize() -> Result<()> { + let workdir = tempfile::tempdir().unwrap(); + let workdir: &Utf8Path = workdir.path().try_into().unwrap(); + let tf = indoc! { r#" + edition: "2024" + "#}; + let finalize_d = workdir.join(FINALIZE_D); + std::fs::create_dir(&finalize_d).unwrap(); + std::fs::write( + finalize_d.join("01-foo"), + indoc::indoc! { r#" + #!/bin/bash + touch foo + sleep 1 + "# }, + )?; + std::fs::write( + finalize_d.join("02-bar"), + indoc::indoc! { r#" + #!/bin/bash + touch bar + "# }, + )?; + for ent in std::fs::read_dir(&finalize_d).unwrap() { + let ent = ent?; + std::fs::set_permissions(ent.path(), fs::Permissions::from_mode(0o755))?; + } + let rootpath = workdir.join("rootfs"); + std::fs::create_dir(&rootpath)?; + let rootpath = Dir::open_ambient_dir(&rootpath, cap_std::ambient_authority())?; + + let tf = new_test_treefile(workdir, tf, None).unwrap(); + assert_eq!(tf.externals.finalize_d.len(), 2); + + tf.exec_finalize_d(&rootpath)?; + + let foometa = rootpath.symlink_metadata("foo").unwrap(); + let barmeta = rootpath.symlink_metadata("bar").unwrap(); + assert!(foometa.modified().unwrap() < barmeta.modified().unwrap()); + + Ok(()) } #[test]