From e324b40050e2272016b72f6bd4b30ccc80d1153c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 21 Jun 2021 10:25:33 -0400 Subject: [PATCH] Add support for modules Add support for composing with modules and for package layering modules on the client-side. This doesn't support modules in `rpm-ostree compose extensions` yet, which is relevant for RHCOS. We can look at adding that in a follow-up. --- docs/treefile.md | 9 ++ rust/src/daemon.rs | 10 ++ rust/src/lib.rs | 3 + rust/src/main.rs | 1 + rust/src/modularity.rs | 130 ++++++++++++++++++++ rust/src/origin.rs | 30 +++++ rust/src/treefile.rs | 81 ++++++++++++ rust/src/utils.rs | 2 +- rust/src/variant_utils.rs | 6 + src/app/libmain.cxx | 6 +- src/app/rpmostree-builtin-status.cxx | 20 +++ src/app/rpmostree-compose-builtin-tree.cxx | 6 + src/daemon/org.projectatomic.rpmostree1.xml | 2 + src/daemon/rpmostree-sysroot-upgrader.cxx | 24 +++- src/daemon/rpmostreed-deployment-utils.cxx | 4 +- src/daemon/rpmostreed-os.cxx | 13 +- src/daemon/rpmostreed-transaction-types.cxx | 75 +++++++++-- src/libpriv/rpmostree-core.cxx | 83 ++++++++++--- src/libpriv/rpmostree-origin.cxx | 104 +++++++++++++++- src/libpriv/rpmostree-origin.h | 22 +++- src/libpriv/rpmostree-util.cxx | 26 +++- src/libpriv/rpmostree-util.h | 4 + tests/common/libtest.sh | 108 ++++++++++++++++ tests/compose/test-basic-unified.sh | 36 ++++++ tests/kolainst/Makefile | 1 + tests/kolainst/destructive/layering-modules | 59 +++++++++ tests/kolainst/kolainst-build.sh | 17 +++ 27 files changed, 844 insertions(+), 38 deletions(-) create mode 100644 rust/src/modularity.rs create mode 100755 tests/kolainst/destructive/layering-modules diff --git a/docs/treefile.md b/docs/treefile.md index d5740baf8d..9d56685f1e 100644 --- a/docs/treefile.md +++ b/docs/treefile.md @@ -79,6 +79,15 @@ It supports the following parameters: * `packages`: Array of strings, required: List of packages to install. * `repo`: String, required: Name of the repo from which to fetch packages. + * `modules`: Object, optional: Describes RPM modules to enable or install. Two + keys are supported: + * `enable`: Array of strings, required: Set of RPM module specs to enable + (the same formats as dnf are supported, e.g. `NAME[:STREAM]`). + One can then cherry-pick specific packages from the enabled modules via + `packages`. + * `install`: Array of strings, required: Set of RPM module specs to install + (the same formats as dnf are supported, e.g. `NAME[:STREAM][/PROFILE]`). + * `ostree-layers`: Array of strings, optional: After all packages are unpacked, check out these OSTree refs, which must already be in the destination repository. Any conflicts with packages will be an error. diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index f05f1fdd53..29fa16ae17 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -81,6 +81,16 @@ fn deployment_populate_variant_origin( // Package mappings. Note these are inserted unconditionally, even if empty. vdict_insert_optvec(&dict, "requested-packages", tf.packages.as_ref()); + vdict_insert_optvec( + &dict, + "requested-modules", + tf.modules.as_ref().map(|m| m.install.as_ref()).flatten(), + ); + vdict_insert_optvec( + &dict, + "modules-enabled", + tf.modules.as_ref().map(|m| m.enable.as_ref()).flatten(), + ); vdict_insert_optmap( &dict, "requested-local-packages", diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 9c50b30afc..0a22ecafd2 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -332,6 +332,8 @@ pub mod ffi { fn get_packages_local(&self) -> Vec; fn get_packages_override_replace_local(&self) -> Vec; fn get_packages_override_remove(&self) -> Vec; + fn get_modules_enable(&self) -> Vec; + fn get_modules_install(&self) -> Vec; fn get_exclude_packages(&self) -> Vec; fn get_install_langs(&self) -> Vec; fn format_install_langs_macro(&self) -> String; @@ -618,6 +620,7 @@ mod lockfile; pub(crate) use self::lockfile::*; mod live; pub(crate) use self::live::*; +pub mod modularity; mod nameservice; mod origin; pub(crate) use self::origin::*; diff --git a/rust/src/main.rs b/rust/src/main.rs index c46ac6904a..866473a91a 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -46,6 +46,7 @@ fn inner_main() -> Result { Some("countme") => rpmostree_rust::countme::entrypoint(&args).map(|_| 0), Some("cliwrap") => rpmostree_rust::cliwrap::entrypoint(&args).map(|_| 0), Some("ex-container") => rpmostree_rust::container::entrypoint(&args).map(|_| 0), + Some("module") => rpmostree_rust::modularity::entrypoint(&args).map(|_| 0), _ => { // Otherwise fall through to C++ main(). Ok(rpmostree_rust::ffi::rpmostree_main(&args)?) diff --git a/rust/src/modularity.rs b/rust/src/modularity.rs new file mode 100644 index 0000000000..fbc910bc2b --- /dev/null +++ b/rust/src/modularity.rs @@ -0,0 +1,130 @@ +//! Implementation of the client-side of "rpm-ostree module". + +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use anyhow::{anyhow, bail, Result}; +use gio::DBusProxyExt; +use ostree_ext::variant_utils; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "rpm-ostree module", no_version)] +#[structopt(rename_all = "kebab-case")] +enum Opt { + /// Enable a module + Enable(InstallOpts), + /// Disable a module + Disable(InstallOpts), + /// Install a module + Install(InstallOpts), + /// Uninstall a module + Uninstall(InstallOpts), +} + +#[derive(Debug, StructOpt)] +struct InstallOpts { + #[structopt(parse(from_str))] + modules: Vec, + #[structopt(long)] + reboot: bool, + #[structopt(long)] + lock_finalization: bool, + #[structopt(long)] + dry_run: bool, +} + +const OPT_KEY_ENABLE_MODULES: &str = "enable-modules"; +const OPT_KEY_DISABLE_MODULES: &str = "disable-modules"; +const OPT_KEY_INSTALL_MODULES: &str = "install-modules"; +const OPT_KEY_UNINSTALL_MODULES: &str = "uninstall-modules"; + +pub fn entrypoint(args: &[&str]) -> Result<()> { + match Opt::from_iter(args.iter().skip(1)) { + Opt::Enable(ref opts) => enable(opts), + Opt::Disable(ref opts) => disable(opts), + Opt::Install(ref opts) => install(opts), + Opt::Uninstall(ref opts) => uninstall(opts), + } +} + +// XXX: Should split out a lot of the below into a more generic Rust wrapper around +// UpdateDeployment() like we have on the C side. + +fn get_modifiers_variant(key: &str, modules: &[String]) -> Result { + let r = glib::VariantDict::new(None); + r.insert_value(key, &crate::variant_utils::new_variant_strv(modules)); + Ok(r.end()) +} + +fn get_options_variant(opts: &InstallOpts) -> Result { + let r = glib::VariantDict::new(None); + r.insert("no-pull-base", &true); + r.insert("reboot", &opts.reboot); + r.insert("lock-finalization", &opts.lock_finalization); + r.insert("dry-run", &opts.dry_run); + Ok(r.end()) +} + +fn enable(opts: &InstallOpts) -> Result<()> { + modules_impl(OPT_KEY_ENABLE_MODULES, opts) +} + +fn disable(opts: &InstallOpts) -> Result<()> { + modules_impl(OPT_KEY_DISABLE_MODULES, opts) +} + +fn install(opts: &InstallOpts) -> Result<()> { + modules_impl(OPT_KEY_INSTALL_MODULES, opts) +} + +fn uninstall(opts: &InstallOpts) -> Result<()> { + modules_impl(OPT_KEY_UNINSTALL_MODULES, opts) +} + +fn modules_impl(key: &str, opts: &InstallOpts) -> Result<()> { + eprintln!("NOTE: Modularity support is experimental and subject to change."); + + if opts.modules.is_empty() { + bail!("At least one module must be specified"); + } + + let client = &mut crate::client::ClientConnection::new()?; + let previous_deployment = client + .get_os_proxy() + .get_cached_property("DefaultDeployment") + .ok_or_else(|| anyhow!("Failed to find default-deployment property"))?; + let modifiers = get_modifiers_variant(key, &opts.modules)?; + let options = get_options_variant(opts)?; + let params = variant_utils::new_variant_tuple(&[modifiers, options]); + let reply = &client.get_os_proxy().call_sync( + "UpdateDeployment", + Some(¶ms), + gio::DBusCallFlags::NONE, + -1, + gio::NONE_CANCELLABLE, + )?; + let reply_child = crate::variant_utils::variant_tuple_get(reply, 0) + .ok_or_else(|| anyhow!("Invalid reply"))?; + let txn_address = reply_child + .get_str() + .ok_or_else(|| anyhow!("Expected string transaction address"))?; + client.transaction_connect_progress_sync(txn_address)?; + let new_deployment = client + .get_os_proxy() + .get_cached_property("DefaultDeployment") + .ok_or_else(|| anyhow!("Failed to find default-deployment property"))?; + if previous_deployment != new_deployment { + unsafe { + crate::ffi::print_treepkg_diff_from_sysroot_path( + "/", + crate::ffi::RpmOstreeDiffPrintFormat::RPMOSTREE_DIFF_PRINT_FORMAT_FULL_MULTILINE, + 0, + std::ptr::null_mut(), + ); + } + } + if opts.dry_run { + println!("Exiting because of '--dry-run' option"); + } + Ok(()) +} diff --git a/rust/src/origin.rs b/rust/src/origin.rs index 49d699dd35..890cef7ba0 100644 --- a/rust/src/origin.rs +++ b/rust/src/origin.rs @@ -19,6 +19,7 @@ use std::result::Result as StdResult; const ORIGIN: &str = "origin"; const RPMOSTREE: &str = "rpmostree"; const PACKAGES: &str = "packages"; +const MODULES: &str = "modules"; const OVERRIDES: &str = "overrides"; /// The set of keys that we parse as BTreeMap and need to ignore ordering changes. @@ -43,6 +44,14 @@ pub(crate) fn origin_to_treefile_inner(kf: &KeyFile) -> Result> { cfg.derive.base_refspec = Some(refspec_str); cfg.packages = parse_stringlist(&kf, PACKAGES, "requested")?; cfg.derive.packages_local = parse_localpkglist(&kf, PACKAGES, "requested-local")?; + let modules_enable = parse_stringlist(&kf, MODULES, "enable")?; + let modules_install = parse_stringlist(&kf, MODULES, "install")?; + if modules_enable.is_some() || modules_install.is_some() { + cfg.modules = Some(crate::treefile::ModulesConfig { + enable: modules_enable, + install: modules_install, + }); + } cfg.derive.override_remove = parse_stringlist(&kf, OVERRIDES, "remove")?; cfg.derive.override_replace_local = parse_localpkglist(&kf, OVERRIDES, "replace-local")?; @@ -143,6 +152,16 @@ fn treefile_to_origin_inner(tf: &Treefile) -> Result { if let Some(pkgs) = tf.derive.override_replace_local.as_ref() { set_sha256_nevra_pkgs(&kf, OVERRIDES, "replace-local", pkgs) } + if let Some(ref modcfg) = tf.modules { + if let Some(modules) = modcfg.enable.as_deref() { + let modules = modules.iter().map(|s| s.as_str()); + kf_set_string_list(&kf, MODULES, "enable", modules) + } + if let Some(modules) = modcfg.install.as_deref() { + let modules = modules.iter().map(|s| s.as_str()); + kf_set_string_list(&kf, MODULES, "install", modules) + } + } // Initramfs bits if let Some(initramfs) = tf.derive.initramfs.as_ref() { @@ -332,6 +351,10 @@ pub(crate) mod test { requested=libvirt;fish; requested-local=4ed748ba060fce4571e7ef19f3f5ed6209f67dbac8327af0d38ea70b96d2f723:foo-1.2-3.x86_64; + [modules] + enable=foo:2.0;bar:rolling; + install=baz:next/development; + [overrides] remove=docker; replace-local=0c7072500af2758e7dc7d7700fed82c3c5f4da7453b4d416e79f75384eee96b0:rpm-ostree-devel-2021.1-2.fc33.x86_64;648ab3ff4d4b708ea180269297de5fa3e972f4481d47b7879c6329272e474d68:rpm-ostree-2021.1-2.fc33.x86_64;8b29b78d0ade6ec3aedb8e3846f036f6f28afe64635d83cb6a034f1004607678:rpm-ostree-libs-2021.1-2.fc33.x86_64; @@ -393,6 +416,13 @@ pub(crate) mod test { tf.parsed.derive.override_commit.unwrap(), "41af286dc0b172ed2f1ca934fd2278de4a1192302ffa07087cea2682e7d372e3" ); + assert_eq!( + tf.parsed.modules, + Some(crate::treefile::ModulesConfig { + enable: Some(vec!["foo:2.0".into(), "bar:rolling".into()]), + install: Some(vec!["baz:next/development".into()]), + }) + ); Ok(()) } diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index b327c30533..2fe4b88618 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -129,6 +129,17 @@ fn treefile_parse_stream( } } + // to be consistent, we also support whitespace-separated modules + if let Some(mut modules) = treefile.modules.take() { + if let Some(enable) = modules.enable.take() { + modules.enable = Some(whitespace_split_packages(&enable)?); + } + if let Some(install) = modules.install.take() { + modules.install = Some(whitespace_split_packages(&install)?); + } + treefile.modules = Some(modules); + } + if let Some(repo_packages) = treefile.repo_packages.take() { treefile.repo_packages = Some( repo_packages @@ -313,6 +324,18 @@ fn merge_hashset_field( } } +/// Merge modules fields. +fn merge_modules(dest: &mut Option, src: &mut Option) { + if let Some(mut srcv) = src.take() { + if let Some(mut destv) = dest.take() { + merge_vec_field(&mut destv.enable, &mut srcv.enable); + merge_vec_field(&mut destv.install, &mut srcv.install); + srcv = destv; + } + *dest = Some(srcv); + } +} + /// Given two configs, merge them. fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) { macro_rules! merge_basics { @@ -384,6 +407,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) { ); merge_basic_field(&mut dest.derive.base_refspec, &mut src.derive.base_refspec); + merge_modules(&mut dest.modules, &mut src.modules); } /// Merge the treefile externals. There are currently only two keys that @@ -595,6 +619,30 @@ impl Treefile { .collect() } + pub(crate) fn get_modules_enable(&self) -> Vec { + self.parsed + .modules + .as_ref() + .map(|m| m.enable.as_ref()) + .flatten() + .map(|v| v.clone()) + .into_iter() + .flatten() + .collect() + } + + pub(crate) fn get_modules_install(&self) -> Vec { + self.parsed + .modules + .as_ref() + .map(|m| m.install.as_ref()) + .flatten() + .map(|v| v.clone()) + .into_iter() + .flatten() + .collect() + } + pub(crate) fn get_packages_override_remove(&self) -> Vec { self.parsed .derive @@ -1097,6 +1145,8 @@ pub(crate) struct TreeComposeConfig { #[serde(skip_serializing_if = "Option::is_none")] #[serde(rename = "repo-packages")] pub(crate) repo_packages: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) modules: Option, // Deprecated option #[serde(skip_serializing_if = "Option::is_none")] pub(crate) bootstrap_packages: Option>, @@ -1224,6 +1274,14 @@ pub(crate) struct RepoPackage { pub(crate) packages: Vec, } +#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)] +pub(crate) struct ModulesConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) enable: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) install: Option>, +} + #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)] pub(crate) struct LegacyTreeComposeConfigFields { #[serde(skip_serializing)] @@ -1426,6 +1484,12 @@ pub(crate) mod tests { - repo: baserepo packages: - blah bloo + modules: + enable: + - foobar:2.0 + install: + - nodejs:15 + - swig:3.0/complete sway:rolling "#}; // This one has "comments" (hence unknown keys) @@ -1720,6 +1784,11 @@ pub(crate) mod tests { - repo: foo2 packages: - qwert + modules: + enable: + - dodo + install: + - bazboo "}, )?; let mut buf = VALID_PRELUDE.to_string(); @@ -1741,6 +1810,18 @@ pub(crate) mod tests { } ]) ); + assert_eq!( + tf.parsed.modules, + Some(ModulesConfig { + enable: Some(vec!["dodo".into(), "foobar:2.0".into()]), + install: Some(vec![ + "bazboo".into(), + "nodejs:15".into(), + "swig:3.0/complete".into(), + "sway:rolling".into(), + ]) + },) + ); Ok(()) } diff --git a/rust/src/utils.rs b/rust/src/utils.rs index 4a396f2194..1779cbf690 100644 --- a/rust/src/utils.rs +++ b/rust/src/utils.rs @@ -223,7 +223,7 @@ pub fn maybe_shell_quote(input: &str) -> String { pub(crate) fn shellsafe_quote(input: Cow) -> Cow { lazy_static! { - static ref SHELLSAFE: Regex = Regex::new("^[[:alnum:]-._/=]+$").unwrap(); + static ref SHELLSAFE: Regex = Regex::new("^[[:alnum:]-._/=:]+$").unwrap(); } if SHELLSAFE.is_match(&input) { diff --git a/rust/src/variant_utils.rs b/rust/src/variant_utils.rs index e7b3f1f398..0b365cda4f 100644 --- a/rust/src/variant_utils.rs +++ b/rust/src/variant_utils.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use glib::translate::*; +use glib::ToVariant; // These constants should really be in gtk-rs lazy_static::lazy_static! { @@ -40,6 +41,11 @@ pub(crate) fn new_variant_array(ty: &glib::VariantTy, children: &[glib::Variant] } } +pub(crate) fn new_variant_strv(strv: &[impl AsRef]) -> glib::Variant { + let v: Vec = strv.iter().map(|s| s.as_ref().to_variant()).collect(); + new_variant_array(&TY_S, &v) +} + pub(crate) fn is_container(v: &glib::Variant) -> bool { unsafe { glib_sys::g_variant_is_container(v.to_glib_none().0) != glib_sys::GFALSE } } diff --git a/src/app/libmain.cxx b/src/app/libmain.cxx index 90d7fa6798..db22a1768e 100644 --- a/src/app/libmain.cxx +++ b/src/app/libmain.cxx @@ -103,10 +103,8 @@ static RpmOstreeCommand commands[] = { /* Rust-implemented commands; they're here so that they show up in `rpm-ostree * --help` alongside the other commands, but the command itself is fully * handled Rust side. */ - /* - { "my-rust-command", static_cast(0), - "Cool thing my command does", NULL }, - */ + { "module", static_cast(0), + "Commands to install/uninstall modules", NULL }, /* Legacy aliases */ { "pkg-add", static_cast(RPM_OSTREE_BUILTIN_FLAG_HIDDEN), NULL, rpmostree_builtin_install }, diff --git a/src/app/rpmostree-builtin-status.cxx b/src/app/rpmostree-builtin-status.cxx index 6b723c6352..8ef53967b4 100644 --- a/src/app/rpmostree-builtin-status.cxx +++ b/src/app/rpmostree-builtin-status.cxx @@ -561,7 +561,10 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy, const gchar *origin_refspec; RpmOstreeRefspecType refspectype = RPMOSTREE_REFSPEC_TYPE_OSTREE; g_autofree const gchar **origin_packages = NULL; + g_autofree const gchar **origin_modules = NULL; + g_autofree const gchar **origin_modules_enabled = NULL; g_autofree const gchar **origin_requested_packages = NULL; + g_autofree const gchar **origin_requested_modules = NULL; g_autofree const gchar **origin_requested_local_packages = NULL; g_autoptr(GVariant) origin_base_removals = NULL; g_autofree const gchar **origin_requested_base_removals = NULL; @@ -572,8 +575,14 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy, { origin_packages = lookup_array_and_canonicalize (dict, "packages"); + origin_modules = + lookup_array_and_canonicalize (dict, "modules"); + origin_modules_enabled = + lookup_array_and_canonicalize (dict, "modules-enabled"); origin_requested_packages = lookup_array_and_canonicalize (dict, "requested-packages"); + origin_requested_modules = + lookup_array_and_canonicalize (dict, "requested-modules"); origin_requested_local_packages = lookup_array_and_canonicalize (dict, "requested-local-packages"); origin_base_removals = @@ -935,10 +944,21 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy, /* requested-packages - packages = inactive (i.e. dormant requests) */ print_packages ("InactiveRequests", max_key_len, origin_requested_packages, origin_packages); + if (origin_requested_modules && opt_verbose) + /* requested-modules - modules = inactive (i.e. dormant requests) */ + /* note the core doesn't support inactive modules yet, but could in the future */ + print_packages ("InactiveModuleRequests", max_key_len, + origin_requested_modules, origin_modules); if (origin_packages) print_packages ("LayeredPackages", max_key_len, origin_packages, NULL); + if (origin_modules) + print_packages ("LayeredModules", max_key_len, + origin_modules, NULL); + if (origin_modules_enabled) + print_packages ("EnabledModules", max_key_len, + origin_modules_enabled, NULL); if (origin_requested_local_packages) print_packages ("LocalPackages", max_key_len, diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 75faeae846..5291854945 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -827,6 +827,12 @@ impl_install_tree (RpmOstreeTreeComposeContext *self, return glnx_throw_errno_prefix (error, "fchdir"); } + /* We don't support installing modules in non-unified mode, because it relies + * on the core writing metadata to the commit metadata (obviously this could + * be supported, but meh...) */ + if (!opt_unified_core && json_object_has_member (self->treefile, "modules")) + return glnx_throw (error, "Composing with modules requires --unified-core"); + /* Read the previous commit. Note we don't actually *need* the full commit; really, only * if one uses `check-passwd: { "type": "previous" }`. There are a few other optimizations * too, e.g. using the previous SELinux policy in unified core. Also, we might need the diff --git a/src/daemon/org.projectatomic.rpmostree1.xml b/src/daemon/org.projectatomic.rpmostree1.xml index 4d37185937..9094e7b5f0 100644 --- a/src/daemon/org.projectatomic.rpmostree1.xml +++ b/src/daemon/org.projectatomic.rpmostree1.xml @@ -298,6 +298,8 @@ "install-packages" (type 'as') "uninstall-packages" (type 'as') "install-local-packages" (type 'ah') + "install-modules" (type 'as') + "uninstall-modules" (type 'as') "override-remove-packages" (type 'as') "override-reset-packages" (type 'as') "override-replace-packages" (type 'as') diff --git a/src/daemon/rpmostree-sysroot-upgrader.cxx b/src/daemon/rpmostree-sysroot-upgrader.cxx index 3c53beebe3..277a2154b0 100644 --- a/src/daemon/rpmostree-sysroot-upgrader.cxx +++ b/src/daemon/rpmostree-sysroot-upgrader.cxx @@ -796,7 +796,6 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self, /* request (owned by origin) --> providing nevra */ g_autoptr(GHashTable) inactive_requests = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free); - g_autoptr(GPtrArray) ret_missing_pkgs = g_ptr_array_new_with_free_func (g_free); /* Add the local pkgs as if they were installed: since they're unconditionally * layered, we treat them as part of the base wrt regular requested pkgs. E.g. @@ -846,8 +845,8 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self, if (matches->len == 0) { - /* no matches, so we'll need to layer it */ - g_ptr_array_add (ret_missing_pkgs, g_strdup (pattern)); + /* no matches, so we'll need to layer it (i.e. not remove it from the + * computed origin) */ continue; } @@ -877,6 +876,18 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self, g_strdup (providing_nevra)); } + /* XXX: Currently, we don't retain information about which modules were + * installed in the commit metadata. So we can't really detect "inactive" + * requests. See related discussions in + * https://github.com/rpm-software-management/libdnf/pull/1207. + * + * In the future, here we could extract e.g. the NSVCAPs from the base + * commit, and mark as inactive all the requests for module names (and + * optionally streams) already installed. + * + * For now, we just implicitly pass through via computed_origin. + */ + if (g_hash_table_size (inactive_requests) > 0) { gboolean changed = FALSE; @@ -969,6 +980,13 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, } else { + /* We still want to prepare() even if there's only enabled modules to validate. + * See comment in rpmostree_origin_may_require_local_assembly(). */ + if (g_hash_table_size (rpmostree_origin_get_modules_enable (self->computed_origin)) > 0) + { + if (!rpmostree_context_prepare (self->ctx, cancellable, error)) + return FALSE; + } rpmostree_context_set_is_empty (self->ctx); self->layering_type = RPMOSTREE_SYSROOT_UPGRADER_LAYERING_LOCAL; } diff --git a/src/daemon/rpmostreed-deployment-utils.cxx b/src/daemon/rpmostreed-deployment-utils.cxx index b0a3dc81b2..48ce02677e 100644 --- a/src/daemon/rpmostreed-deployment-utils.cxx +++ b/src/daemon/rpmostreed-deployment-utils.cxx @@ -248,10 +248,11 @@ rpmostreed_deployment_generate_variant (OstreeSysroot *sysroot, gboolean is_layered = FALSE; g_autofree char *base_checksum = NULL; g_auto(GStrv) layered_pkgs = NULL; + g_auto(GStrv) layered_modules = NULL; g_autoptr(GVariant) removed_base_pkgs = NULL; g_autoptr(GVariant) replaced_base_pkgs = NULL; if (!rpmostree_deployment_get_layered_info (repo, deployment, &is_layered, NULL, - &base_checksum, &layered_pkgs, + &base_checksum, &layered_pkgs, &layered_modules, &removed_base_pkgs, &replaced_base_pkgs, error)) return NULL; @@ -340,6 +341,7 @@ rpmostreed_deployment_generate_variant (OstreeSysroot *sysroot, } g_variant_dict_insert (dict, "packages", "^as", layered_pkgs); + g_variant_dict_insert (dict, "modules", "^as", layered_modules); g_variant_dict_insert_value (dict, "base-removals", removed_base_pkgs); g_variant_dict_insert_value (dict, "base-local-replacements", replaced_base_pkgs); diff --git a/src/daemon/rpmostreed-os.cxx b/src/daemon/rpmostreed-os.cxx index 2bfe67e44d..a16501f75f 100644 --- a/src/daemon/rpmostreed-os.cxx +++ b/src/daemon/rpmostreed-os.cxx @@ -172,6 +172,14 @@ os_authorize_method (GDBusInterfaceSkeleton *interface, vardict_lookup_strv (&modifiers_dict, "install-packages"); g_autofree char **uninstall_pkgs = vardict_lookup_strv (&modifiers_dict, "uninstall-packages"); + g_autofree char **enable_modules = + vardict_lookup_strv (&modifiers_dict, "enable-modules"); + g_autofree char **disable_modules = + vardict_lookup_strv (&modifiers_dict, "disable-modules"); + g_autofree char **install_modules = + vardict_lookup_strv (&modifiers_dict, "install-modules"); + g_autofree char **uninstall_modules = + vardict_lookup_strv (&modifiers_dict, "uninstall-modules"); g_autofree const char *const *override_replace_pkgs = vardict_lookup_strv (&modifiers_dict, "override-replace-packages"); g_autofree const char *const *override_remove_pkgs = @@ -201,7 +209,10 @@ os_authorize_method (GDBusInterfaceSkeleton *interface, else if (!no_pull_base) g_ptr_array_add (actions, (void*)"org.projectatomic.rpmostree1.upgrade"); - if (install_pkgs != NULL || uninstall_pkgs != NULL || no_layering) + if (install_pkgs != NULL || uninstall_pkgs != NULL || + enable_modules != NULL || disable_modules != NULL || + install_modules != NULL || uninstall_modules != NULL || + no_layering) g_ptr_array_add (actions, (void*)"org.projectatomic.rpmostree1.install-uninstall-packages"); if (install_local_pkgs != NULL && g_variant_n_children (install_local_pkgs) > 0) diff --git a/src/daemon/rpmostreed-transaction-types.cxx b/src/daemon/rpmostreed-transaction-types.cxx index ac6b5cd2b3..b46578bf81 100644 --- a/src/daemon/rpmostreed-transaction-types.cxx +++ b/src/daemon/rpmostreed-transaction-types.cxx @@ -601,6 +601,10 @@ typedef struct { char **install_pkgs; /* strv but strings owned by modifiers */ GUnixFDList *install_local_pkgs; char **uninstall_pkgs; /* strv but strings owned by modifiers */ + char **enable_modules; /* strv but strings owned by modifiers */ + char **disable_modules; /* strv but strings owned by modifiers */ + char **install_modules; /* strv but strings owned by modifiers */ + char **uninstall_modules; /* strv but strings owned by modifiers */ char **override_replace_pkgs; /* strv but strings owned by modifiers */ GUnixFDList *override_replace_local_pkgs; char **override_remove_pkgs; /* strv but strings owned by modifiers */ @@ -630,6 +634,10 @@ deploy_transaction_finalize (GObject *object) g_free (self->install_pkgs); g_clear_pointer (&self->install_local_pkgs, g_object_unref); g_free (self->uninstall_pkgs); + g_free (self->enable_modules); + g_free (self->disable_modules); + g_free (self->install_modules); + g_free (self->uninstall_modules); g_free (self->override_replace_pkgs); g_clear_pointer (&self->override_replace_local_pkgs, g_object_unref); g_free (self->override_remove_pkgs); @@ -900,7 +908,7 @@ deploy_transaction_execute (RpmostreedTransaction *transaction, const gboolean idempotent_layering = deploy_has_bool_option (self, "idempotent-layering"); const gboolean download_only = ((self->flags & RPMOSTREE_TRANSACTION_DEPLOY_FLAG_DOWNLOAD_ONLY) > 0); - /* Mainly for the `install` and `override` commands */ + /* Mainly for the `install`, `module install`, and `override` commands */ const gboolean no_pull_base = ((self->flags & RPMOSTREE_TRANSACTION_DEPLOY_FLAG_NO_PULL_BASE) > 0); /* Used to background check for updates; this essentially means downloading the minimum @@ -929,7 +937,7 @@ deploy_transaction_execute (RpmostreedTransaction *transaction, no_overrides); if (!is_override) { - if (self->install_pkgs || self->install_local_pkgs) + if (self->install_pkgs || self->install_local_pkgs || self->install_modules) is_install = TRUE; else is_uninstall = TRUE; @@ -980,12 +988,24 @@ deploy_transaction_execute (RpmostreedTransaction *transaction, if (self->uninstall_pkgs) g_string_append_printf (txn_title, "; uninstall: %u", g_strv_length (self->uninstall_pkgs)); + if (self->disable_modules) + g_string_append_printf (txn_title, "; module disable: %u", + g_strv_length (self->disable_modules)); + if (self->uninstall_modules) + g_string_append_printf (txn_title, "; module uninstall: %u", + g_strv_length (self->uninstall_modules)); if (self->install_pkgs) g_string_append_printf (txn_title, "; install: %u", g_strv_length (self->install_pkgs)); if (self->install_local_pkgs) g_string_append_printf (txn_title, "; localinstall: %u", g_unix_fd_list_get_length (self->install_local_pkgs)); + if (self->enable_modules) + g_string_append_printf (txn_title, "; module enable: %u", + g_strv_length (self->enable_modules)); + if (self->install_modules) + g_string_append_printf (txn_title, "; module install: %u", + g_strv_length (self->install_modules)); rpmostree_transaction_set_title (RPMOSTREE_TRANSACTION (transaction), txn_title->str); } @@ -1148,11 +1168,30 @@ deploy_transaction_execute (RpmostreedTransaction *transaction, if (!rpmostree_origin_remove_all_packages (origin, &remove_changed, error)) return FALSE; } - else if (self->uninstall_pkgs) + else { - if (!rpmostree_origin_remove_packages (origin, self->uninstall_pkgs, - idempotent_layering, &remove_changed, error)) - return FALSE; + gboolean local_changed = FALSE; + if (self->uninstall_pkgs) + { + if (!rpmostree_origin_remove_packages (origin, self->uninstall_pkgs, + idempotent_layering, &local_changed, error)) + return FALSE; + } + remove_changed = remove_changed || local_changed; + if (self->disable_modules) + { + if (!rpmostree_origin_remove_modules (origin, self->disable_modules, + TRUE, &local_changed, error)) + return FALSE; + } + remove_changed = remove_changed || local_changed; + if (self->uninstall_modules) + { + if (!rpmostree_origin_remove_modules (origin, self->uninstall_modules, + FALSE, &local_changed, error)) + return FALSE; + } + remove_changed = remove_changed || local_changed; } /* In reality, there may not be any new layer required even if `remove_changed` is TRUE @@ -1203,6 +1242,24 @@ deploy_transaction_execute (RpmostreedTransaction *transaction, changed = changed || add_changed; } + if (self->enable_modules) + { + gboolean add_changed = FALSE; + if (!rpmostree_origin_add_modules (origin, self->enable_modules, TRUE, &add_changed, error)) + return FALSE; + + changed = changed || add_changed; + } + + if (self->install_modules) + { + gboolean add_changed = FALSE; + if (!rpmostree_origin_add_modules (origin, self->install_modules, FALSE, &add_changed, error)) + return FALSE; + + changed = changed || add_changed; + } + if (self->install_local_pkgs != NULL) { g_autoptr(GPtrArray) pkgs = NULL; @@ -1242,7 +1299,7 @@ deploy_transaction_execute (RpmostreedTransaction *transaction, g_autoptr(GVariant) removed = NULL; g_autoptr(GVariant) replaced = NULL; if (!rpmostree_deployment_get_layered_info (repo, merge_deployment, NULL, NULL, NULL, - NULL, &removed, &replaced, error)) + NULL, NULL, &removed, &replaced, error)) return FALSE; g_autoptr(GHashTable) nevra_to_name = g_hash_table_new (g_str_hash, g_str_equal); @@ -1756,6 +1813,10 @@ rpmostreed_transaction_new_deploy (GDBusMethodInvocation *invocation, self->override_replace_pkgs = vardict_lookup_strv_canonical (self->modifiers, "override-replace-packages"); self->override_remove_pkgs = vardict_lookup_strv_canonical (self->modifiers, "override-remove-packages"); self->override_reset_pkgs = vardict_lookup_strv_canonical (self->modifiers, "override-reset-packages"); + self->enable_modules = vardict_lookup_strv_canonical (self->modifiers, "enable-modules"); + self->disable_modules = vardict_lookup_strv_canonical (self->modifiers, "disable-modules"); + self->install_modules = vardict_lookup_strv_canonical (self->modifiers, "install-modules"); + self->uninstall_modules = vardict_lookup_strv_canonical (self->modifiers, "uninstall-modules"); /* default to allowing downgrades for rebases & deploys (without --disallow-downgrade) */ if (vardict_lookup_bool (self->options, "allow-downgrade", refspec_or_revision)) diff --git a/src/libpriv/rpmostree-core.cxx b/src/libpriv/rpmostree-core.cxx index 4019299f60..b27be5a5cd 100644 --- a/src/libpriv/rpmostree-core.cxx +++ b/src/libpriv/rpmostree-core.cxx @@ -567,6 +567,26 @@ rpmostree_context_setup (RpmOstreeContext *self, if (!dnf_context_setup (self->dnfctx, cancellable, error)) return FALSE; + /* XXX: If we have modules to install, then we need libdnf to handle it, and + * we can't avoid not parsing repodata because modules are entirely a repodata + * concept. So for now, force off pkgcache-only. This means that e.g. client + * side operations that are normally cache-only like `rpm-ostree uninstall` + * will still try to fetch metadata, and might install newer versions of other + * packages... we can probably hack that in the future. */ + if (self->pkgcache_only) + { + gboolean disable_cacheonly = FALSE; + auto modules_enable = self->treefile_rs->get_modules_enable(); + disable_cacheonly = disable_cacheonly || !modules_enable.empty(); + auto modules_install = self->treefile_rs->get_modules_install(); + disable_cacheonly = disable_cacheonly || !modules_install.empty(); + if (disable_cacheonly) + { + self->pkgcache_only = FALSE; + sd_journal_print (LOG_WARNING, "Ignoring pkgcache-only request in presence of module requests"); + } + } + /* disable all repos in pkgcache-only mode, otherwise obey "repos" key */ if (self->pkgcache_only) { @@ -985,22 +1005,10 @@ rpmostree_context_download_metadata (RpmOstreeContext *self, g_signal_handler_disconnect (hifstate, progress_sigid); } - /* For now, we don't natively support modules. But we still want to be able to install - * modular packages if the repos are enabled, but libdnf automatically filters them out. - * So for now, let's tell libdnf that we do want to be able to see them. See: - * https://github.com/projectatomic/rpm-ostree/issues/1435 */ - dnf_sack_set_module_excludes (dnf_context_get_sack (self->dnfctx), NULL); - /* And also mark all repos as hotfix repos so that we can indiscriminately cherry-pick - * from modular repos and non-modular repos alike. */ - g_autoptr(GPtrArray) repos = - rpmostree_get_enabled_rpmmd_repos (self->dnfctx, DNF_REPO_ENABLED_PACKAGES); - for (guint i = 0; i < repos->len; i++) - dnf_repo_set_module_hotfixes (static_cast(repos->pdata[i]), TRUE); - // Print repo information - for (guint i = 0; i < repos->len; i++) + for (guint i = 0; i < rpmmd_repos->len; i++) { - auto repo = static_cast(repos->pdata[i]); + auto repo = static_cast(rpmmd_repos->pdata[i]); gboolean updated = g_hash_table_contains (updated_repos, repo); guint64 ts = dnf_repo_get_timestamp_generated (repo); g_autofree char *repo_ts_str = rpmostree_timestamp_str_from_unix_utc (ts); @@ -1705,6 +1713,8 @@ rpmostree_context_prepare (RpmOstreeContext *self, auto packages_override_replace_local = self->treefile_rs->get_packages_override_replace_local(); auto packages_override_remove = self->treefile_rs->get_packages_override_remove(); auto exclude_packages = self->treefile_rs->get_exclude_packages (); + auto modules_enable = self->treefile_rs->get_modules_enable(); + auto modules_install = self->treefile_rs->get_modules_install(); /* we only support pure installs for now (compose case) */ if (self->lockfile) @@ -1883,8 +1893,11 @@ rpmostree_context_prepare (RpmOstreeContext *self, hy_goal_install (goal, pkg); /* Now repo-packages; only supported during server composes for now. */ + g_autoptr(DnfPackageSet) pinned_pkgs = NULL; if (!self->is_system) { + pinned_pkgs = dnf_packageset_new (sack); + Map *pinned_pkgs_map = dnf_packageset_get_map (pinned_pkgs); auto repo_pkgs = self->treefile_rs->get_repo_packages(); for (auto & repo_pkg : repo_pkgs) { @@ -1907,10 +1920,43 @@ rpmostree_context_prepare (RpmOstreeContext *self, hy_selector_pkg_set (selector, pset); if (!hy_goal_install_selector (goal, selector, error)) return FALSE; + + map_or (pinned_pkgs_map, dnf_packageset_get_map (pset)); } } } + gboolean we_got_modules = FALSE; + if (!modules_enable.empty()) + { + g_auto(GStrv) modules = rpmostree_cxx_string_vec_to_strv (modules_enable); + if (!dnf_context_module_enable (dnfctx, (const char**)modules, error)) + return FALSE; + we_got_modules = TRUE; + } + + if (!modules_install.empty()) + { + g_auto(GStrv) modules = rpmostree_cxx_string_vec_to_strv (modules_install); + if (!dnf_context_module_install (dnfctx, (const char**)modules, error)) + return glnx_prefix_error (error, "Installing modules"); + we_got_modules = TRUE; + } + + /* By default, when enabling a module, trying to install a package "foo" will + * always prioritize the "foo" in the module. This is what we want, but in the + * case of pinned repo packages, we want to be able to override that. So we + * need to fiddle with the modular excludes. */ + if (we_got_modules && pinned_pkgs && dnf_packageset_count (pinned_pkgs) > 0) + { + g_autoptr(DnfPackageSet) excludes = dnf_sack_get_module_excludes (sack); + g_autoptr(DnfPackageSet) cloned_pkgs = dnf_packageset_clone (pinned_pkgs); + Map *m = dnf_packageset_get_map (cloned_pkgs); + map_invertall (m); + map_and (dnf_packageset_get_map (excludes), m); + dnf_sack_set_module_excludes (sack, excludes); + } + /* And finally, handle repo packages to install */ g_autoptr(GPtrArray) missing_pkgs = NULL; for (auto &pkgname_v : packages) @@ -4303,6 +4349,13 @@ rpmostree_context_commit (RpmOstreeContext *self, } g_variant_builder_add (&metadata_builder, "{sv}", "rpmostree.packages", g_variant_builder_end (pkgs_v)); + /* embed modules layered */ + auto modules = self->treefile_rs->get_modules_install(); + auto modules_v = g_variant_builder_new (G_VARIANT_TYPE ("as")); + for (auto &mod: modules) + g_variant_builder_add (modules_v, "s", mod.c_str()); + g_variant_builder_add (&metadata_builder, "{sv}", "rpmostree.modules", g_variant_builder_end (modules_v)); + /* embed packages removed */ /* we have to embed both the pkgname and the full nevra to make it easier to match * them up with origin directives. the full nevra is used for status -v */ @@ -4339,7 +4392,7 @@ rpmostree_context_commit (RpmOstreeContext *self, /* be nice to our future selves */ g_variant_builder_add (&metadata_builder, "{sv}", "rpmostree.clientlayer_version", - g_variant_new_uint32 (4)); + g_variant_new_uint32 (5)); } else if (assemble_type == RPMOSTREE_ASSEMBLE_TYPE_SERVER_BASE) { diff --git a/src/libpriv/rpmostree-origin.cxx b/src/libpriv/rpmostree-origin.cxx index 0531cce692..90210ae574 100644 --- a/src/libpriv/rpmostree-origin.cxx +++ b/src/libpriv/rpmostree-origin.cxx @@ -49,6 +49,8 @@ struct RpmOstreeOrigin { char **cached_initramfs_args; GHashTable *cached_initramfs_etc_files; /* set of paths */ GHashTable *cached_packages; /* set of reldeps */ + GHashTable *cached_modules_enable; /* set of module specs to enable */ + GHashTable *cached_modules_install; /* set of module specs to install */ GHashTable *cached_local_packages; /* NEVRA --> header sha256 */ /* GHashTable *cached_overrides_replace; XXX: NOT IMPLEMENTED YET */ GHashTable *cached_overrides_local_replace; /* NEVRA --> header sha256 */ @@ -107,6 +109,8 @@ rpmostree_origin_parse_keyfile (GKeyFile *origin, ret->kf = keyfile_dup (origin); ret->cached_packages = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + ret->cached_modules_enable = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + ret->cached_modules_install = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); ret->cached_local_packages = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); ret->cached_overrides_local_replace = @@ -170,6 +174,14 @@ rpmostree_origin_parse_keyfile (GKeyFile *origin, ret->cached_local_packages, error)) return FALSE; + if (!parse_packages_strv (ret->kf, "modules", "enable", FALSE, + ret->cached_modules_enable, error)) + return FALSE; + + if (!parse_packages_strv (ret->kf, "modules", "install", FALSE, + ret->cached_modules_install, error)) + return FALSE; + if (!parse_packages_strv (ret->kf, "overrides", "remove", FALSE, ret->cached_overrides_remove, error)) return FALSE; @@ -271,6 +283,18 @@ rpmostree_origin_get_packages (RpmOstreeOrigin *origin) return origin->cached_packages; } +GHashTable * +rpmostree_origin_get_modules_enable (RpmOstreeOrigin *origin) +{ + return origin->cached_modules_enable; +} + +GHashTable * +rpmostree_origin_get_modules_install (RpmOstreeOrigin *origin) +{ + return origin->cached_modules_install; +} + GHashTable * rpmostree_origin_get_local_packages (RpmOstreeOrigin *origin) { @@ -332,18 +356,24 @@ rpmostree_origin_may_require_local_assembly (RpmOstreeOrigin *origin) rpmostree_origin_get_cliwrap (origin) || rpmostree_origin_get_regenerate_initramfs (origin) || (g_hash_table_size (origin->cached_initramfs_etc_files) > 0) || - rpmostree_origin_has_packages (origin); + rpmostree_origin_has_packages (origin) || + /* Technically, alone it doesn't require require assembly, but it still + * requires fetching repo metadata to validate (remember: modules are a + * pure rpmmd concept). This means we may pay the cost of an unneeded + * tree checkout, but it's not worth trying to optimize for it. */ + (g_hash_table_size (origin->cached_modules_enable) > 0); } /* Returns TRUE if this origin contains overlay or override packages */ gboolean rpmostree_origin_has_packages (RpmOstreeOrigin *origin) { - return + return (g_hash_table_size (origin->cached_packages) > 0) || (g_hash_table_size (origin->cached_local_packages) > 0) || (g_hash_table_size (origin->cached_overrides_local_replace) > 0) || - (g_hash_table_size (origin->cached_overrides_remove) > 0); + (g_hash_table_size (origin->cached_overrides_remove) > 0) || + (g_hash_table_size (origin->cached_modules_install) > 0); } GKeyFile * @@ -381,6 +411,8 @@ rpmostree_origin_unref (RpmOstreeOrigin *origin) g_free (origin->cached_unconfigured_state); g_strfreev (origin->cached_initramfs_args); g_clear_pointer (&origin->cached_packages, g_hash_table_unref); + g_clear_pointer (&origin->cached_modules_enable, g_hash_table_unref); + g_clear_pointer (&origin->cached_modules_install, g_hash_table_unref); g_clear_pointer (&origin->cached_local_packages, g_hash_table_unref); g_clear_pointer (&origin->cached_overrides_local_replace, g_hash_table_unref); g_clear_pointer (&origin->cached_overrides_remove, g_hash_table_unref); @@ -815,6 +847,50 @@ rpmostree_origin_remove_packages (RpmOstreeOrigin *origin, return TRUE; } +gboolean +rpmostree_origin_add_modules (RpmOstreeOrigin *origin, + char **modules, + gboolean enable_only, + gboolean *out_changed, + GError **error) +{ + const char *key = enable_only ? "enable" : "install"; + GHashTable *target = enable_only ? origin->cached_modules_enable + : origin->cached_modules_install; + gboolean changed = FALSE; + for (char **mod = modules; mod && *mod; mod++) + changed = (g_hash_table_add (target, g_strdup (*mod)) || changed); + + if (changed) + update_string_list_from_hash_table (origin->kf, "modules", key, target); + if (out_changed) + *out_changed = changed; + + return TRUE; +} + +gboolean +rpmostree_origin_remove_modules (RpmOstreeOrigin *origin, + char **modules, + gboolean enable_only, + gboolean *out_changed, + GError **error) +{ + const char *key = enable_only ? "enable" : "install"; + GHashTable *target = enable_only ? origin->cached_modules_enable + : origin->cached_modules_install; + gboolean changed = FALSE; + for (char **mod = modules; mod && *mod; mod++) + changed = (g_hash_table_remove (target, *mod) || changed); + + if (changed) + update_string_list_from_hash_table (origin->kf, "modules", key, target); + if (out_changed) + *out_changed = changed; + + return TRUE; +} + gboolean rpmostree_origin_remove_all_packages (RpmOstreeOrigin *origin, gboolean *out_changed, @@ -822,6 +898,8 @@ rpmostree_origin_remove_all_packages (RpmOstreeOrigin *origin, { gboolean changed = FALSE; gboolean local_changed = FALSE; + gboolean modules_enable_changed = FALSE; + gboolean modules_install_changed = FALSE; if (g_hash_table_size (origin->cached_packages) > 0) { @@ -835,14 +913,32 @@ rpmostree_origin_remove_all_packages (RpmOstreeOrigin *origin, local_changed = TRUE; } + if (g_hash_table_size (origin->cached_modules_enable) > 0) + { + g_hash_table_remove_all (origin->cached_modules_enable); + modules_enable_changed = TRUE; + } + + if (g_hash_table_size (origin->cached_modules_install) > 0) + { + g_hash_table_remove_all (origin->cached_modules_install); + modules_install_changed = TRUE; + } + if (changed) update_keyfile_pkgs_from_cache (origin, "packages", "requested", origin->cached_packages, FALSE); if (local_changed) update_keyfile_pkgs_from_cache (origin, "packages", "requested-local", origin->cached_local_packages, TRUE); + if (modules_enable_changed) + update_keyfile_pkgs_from_cache (origin, "modules", "enable", + origin->cached_modules_enable, FALSE); + if (modules_install_changed) + update_keyfile_pkgs_from_cache (origin, "modules", "install", + origin->cached_modules_install, FALSE); if (out_changed) - *out_changed = changed || local_changed; + *out_changed = changed || local_changed || modules_enable_changed || modules_install_changed; return TRUE; } diff --git a/src/libpriv/rpmostree-origin.h b/src/libpriv/rpmostree-origin.h index 56d81cf382..b7325bb261 100644 --- a/src/libpriv/rpmostree-origin.h +++ b/src/libpriv/rpmostree-origin.h @@ -80,6 +80,12 @@ rpmostree_origin_get_custom_description (RpmOstreeOrigin *origin, GHashTable * rpmostree_origin_get_packages (RpmOstreeOrigin *origin); +GHashTable * +rpmostree_origin_get_modules_enable (RpmOstreeOrigin *origin); + +GHashTable * +rpmostree_origin_get_modules_install (RpmOstreeOrigin *origin); + GHashTable * rpmostree_origin_get_local_packages (RpmOstreeOrigin *origin); @@ -181,6 +187,20 @@ rpmostree_origin_remove_all_packages (RpmOstreeOrigin *origin, gboolean *out_changed, GError **error); +gboolean +rpmostree_origin_add_modules (RpmOstreeOrigin *origin, + char **modules, + gboolean enable_only, + gboolean *out_changed, + GError **error); + +gboolean +rpmostree_origin_remove_modules (RpmOstreeOrigin *origin, + char **modules, + gboolean enable_only, + gboolean *out_changed, + GError **error); + typedef enum { /* RPMOSTREE_ORIGIN_OVERRIDE_REPLACE, */ RPMOSTREE_ORIGIN_OVERRIDE_REPLACE_LOCAL, @@ -203,4 +223,4 @@ rpmostree_origin_remove_all_overrides (RpmOstreeOrigin *origin, gboolean *out_changed, GError **error); -G_END_DECLS \ No newline at end of file +G_END_DECLS diff --git a/src/libpriv/rpmostree-util.cxx b/src/libpriv/rpmostree-util.cxx index 11aa5de60e..68483bfcfa 100644 --- a/src/libpriv/rpmostree-util.cxx +++ b/src/libpriv/rpmostree-util.cxx @@ -511,6 +511,7 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, guint *out_layer_version, char **out_base_layer, char ***out_layered_pkgs, + char ***out_layered_modules, GVariant **out_removed_base_pkgs, GVariant **out_replaced_base_pkgs, GError **error) @@ -527,6 +528,7 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, /* only fetch pkgs if we have to */ g_auto(GStrv) layered_pkgs = NULL; + g_auto(GStrv) layered_modules = NULL; g_autoptr(GVariant) removed_base_pkgs = NULL; g_autoptr(GVariant) replaced_base_pkgs = NULL; if (layeredmeta.is_layered && (out_layered_pkgs != NULL || out_removed_base_pkgs != NULL)) @@ -567,6 +569,12 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, G_VARIANT_TYPE ("a(vv)")); g_assert (replaced_base_pkgs); } + + if (layeredmeta.clientlayer_version >= 5) + { + g_assert (g_variant_dict_lookup (dict, "rpmostree.modules", "^as", + &layered_modules)); + } } /* canonicalize outputs to empty array */ @@ -583,6 +591,12 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, layered_pkgs = g_new0 (char*, 1); *out_layered_pkgs = util::move_nullify (layered_pkgs); } + if (out_layered_modules != NULL) + { + if (!layered_modules) + layered_modules = g_new0 (char*, 1); + *out_layered_modules = util::move_nullify (layered_modules); + } if (out_removed_base_pkgs != NULL) { if (!removed_base_pkgs) @@ -609,7 +623,7 @@ rpmostree_deployment_get_base_layer (OstreeRepo *repo, GError **error) { return rpmostree_deployment_get_layered_info (repo, deployment, NULL, NULL, - out_base_layer, NULL, NULL, NULL, error); + out_base_layer, NULL, NULL, NULL, NULL, error); } static gboolean @@ -1162,3 +1176,13 @@ rpmostree_variant_native_to_be (GVariant **v) { rpmostree_variant_be_to_native (v); } + +char** +rpmostree_cxx_string_vec_to_strv (rust::Vec &v) +{ + g_autoptr(GPtrArray) r = g_ptr_array_new_with_free_func (g_free); + for (auto & s : v) + g_ptr_array_add (r, g_strdup (s.c_str())); + g_ptr_array_add (r, NULL); + return (char**)g_ptr_array_free (util::move_nullify(r), FALSE); +} diff --git a/src/libpriv/rpmostree-util.h b/src/libpriv/rpmostree-util.h index d94bc960bd..0871a207e1 100644 --- a/src/libpriv/rpmostree-util.h +++ b/src/libpriv/rpmostree-util.h @@ -245,6 +245,7 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, guint *out_layer_version, char **out_base_layer, char ***out_layered_pkgs, + char ***out_layered_modules, GVariant **out_removed_base_pkgs, GVariant **out_replaced_base_pkgs, GError **error); @@ -338,4 +339,7 @@ rpmostree_variant_be_to_native (GVariant **v); void rpmostree_variant_native_to_be (GVariant **v); +char** +rpmostree_cxx_string_vec_to_strv (rust::Vec &v); + G_END_DECLS diff --git a/tests/common/libtest.sh b/tests/common/libtest.sh index 10a77eec69..a661d56ddf 100644 --- a/tests/common/libtest.sh +++ b/tests/common/libtest.sh @@ -305,6 +305,114 @@ EOF files "${install_dir}/${name}.pp" } +# build a module +# $1 - module name +# $2+ - optional, treated as directive/value pairs +build_module() { + # warning: this function is at the boundary of comfortable bash scripting + local name=$1; shift + local stream=latest + local version=$(date +%Y%m%d%H%M%S) + local context=$(uuidgen | cut -d- -f1) + local arch=x86_64 + + declare -A profiles + declare -a rpms + local requires= + while [ $# -ne 0 ]; do + local section=$1; shift + local arg=$1; shift + case $section in + stream|version|context|arch|requires) + declare $section="$arg";; + rpm) + rpms+=($arg);; + profile) + profiles[${arg%:*}]=${arg#*:};; + *) + assert_not_reached "unhandled section $section";; + esac + done + + local rendered_profiles="{" + for profile in "${!profiles[@]}"; do + rendered_profiles+="$profile: {rpms: [${profiles[$profile]}]}," + done + rendered_profiles+="}" + + # write out the module YAML + # see https://docs.fedoraproject.org/en-US/modularity/building-modules/fedora/defining-modules/ + mkdir -p $test_tmpdir/yumrepo/modules + local md=$test_tmpdir/yumrepo/modules/$name:$stream:$version:$context:$arch.modulemd.yaml + cat >> $md << EOF +document: modulemd +version: 2 +data: + name: $name + stream: $stream + version: $version + context: $context + arch: $arch + summary: $name + description: $name + license: {module: [MIT]} + dependencies: + - requires: {platform: []${requires:+, ${requires}}} + profiles: $rendered_profiles + artifacts: + rpms: [$(IFS=,; echo "${rpms[*]}")] +EOF + + # use --keep-all-metadata to retain previous updateinfo + (cd $test_tmpdir/yumrepo && + createrepo_c --no-database --update --keep-all-metadata .) +} + +# build a module defaults file +# $1 - module name +# $2+ - optional, treated as directive/value pairs +build_module_defaults() { + local name=$1; shift + declare -A defprofiles + local defstream= + while [ $# -ne 0 ]; do + local section=$1; shift + local arg=$1; shift + case $section in + defstream) + defstream="$arg";; + defprofile) + defprofiles[${arg%:*}]=${arg#*:};; + *) + assert_not_reached "unhandled section $section";; + esac + done + + local rendered_dict="{" + for stream in "${!defprofiles[@]}"; do + rendered_dict+="$stream: [${defprofiles[$stream]}]," + done + rendered_dict+="}" + + # write out the module defaults YAML + # see https://github.com/fedora-modularity/libmodulemd/blob/8577d670e88b9a0938428df80bfbdfe6b69698de/yaml_specs/modulemd_defaults_v1.yaml + mkdir -p $test_tmpdir/yumrepo/modules + local md=$test_tmpdir/yumrepo/modules/$name.modulemd-defaults.yaml + cat >> $md << EOF +document: modulemd-defaults +version: 1 +data: + module: $name + modified: $(date +%Y%m%d%H%M) + ${defstream:+stream: ${defstream}} + profiles: ${rendered_dict} +EOF + + # use --keep-all-metadata to retain previous updateinfo + (cd $test_tmpdir/yumrepo && + createrepo_c --no-database --update --keep-all-metadata .) +} + files_are_hardlinked() { inode1=$(stat -c %i $1) inode2=$(stat -c %i $2) diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index de7b2815cc..febafebb30 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -26,6 +26,33 @@ tf['repo-packages'] = [{ }] " +treefile_pyedit "tf['modules'] = { + 'enable': [], + 'install': [], +}" + +build_rpm foomodular requires foomodular-ext +build_rpm foomodular-ext +build_rpm foomodular-optional +build_module foomodular \ + stream mystream \ + profile myprof:foomodular \ + rpm foomodular-0:1.0-1.x86_64 \ + rpm foomodular-ext-0:1.0-1.x86_64 \ + rpm foomodular-optional-0:1.0-1.x86_64 +treefile_pyedit "tf['modules']['install'] += ['foomodular:mystream/myprof']" + +build_rpm barmodular requires barmodular-ext +build_rpm barmodular-ext +build_rpm barmodular-optional +build_module barmodular \ + stream latest \ + rpm barmodular-0:1.0-1.x86_64 \ + rpm barmodular-ext-0:1.0-1.x86_64 \ + rpm barmodular-optional-0:1.0-1.x86_64 +treefile_pyedit "tf['modules']['enable'] += ['barmodular:latest']" +treefile_append "packages" '["barmodular"]' + # Test --print-only. We also # just in this test (for now) use ${basearch} to test substitution. # shellcheck disable=SC2016 @@ -123,6 +150,15 @@ assert_not_file_has_content_literal db-diff-adv.txt TEST-SEC-LOW assert_file_has_content_literal db-diff-adv.txt TEST-SEC-CRIT echo "ok db diff --advisories" +rpm-ostree db list --repo="${repo}" "${treeref}" > db-list.txt +assert_file_has_content_literal db-list.txt foomodular-1.0-1.x86_64 +assert_file_has_content_literal db-list.txt foomodular-ext-1.0-1.x86_64 +assert_not_file_has_content_literal db-list.txt foomodular-optional +assert_file_has_content_literal db-list.txt barmodular-1.0-1.x86_64 +assert_file_has_content_literal db-list.txt barmodular-ext-1.0-1.x86_64 +assert_not_file_has_content_literal db-list.txt barmodular-optional +echo "ok modules" + build_rpm dodo-base build_rpm dodo requires dodo-base build_rpm solitaire diff --git a/tests/kolainst/Makefile b/tests/kolainst/Makefile index 85a2485bb4..f63bffbf18 100644 --- a/tests/kolainst/Makefile +++ b/tests/kolainst/Makefile @@ -18,4 +18,5 @@ install: all rsync -prlv rpm-repos/ $(KOLA_TESTDIR)/destructive/data/rpm-repos/ localinstall: all + rm -rf ../kola make install KOLA_TESTDIR=../kola diff --git a/tests/kolainst/destructive/layering-modules b/tests/kolainst/destructive/layering-modules new file mode 100755 index 0000000000..e2fcf9be7d --- /dev/null +++ b/tests/kolainst/destructive/layering-modules @@ -0,0 +1,59 @@ +#!/bin/bash +set -euo pipefail + +. ${KOLA_EXT_DATA}/libtest.sh +cd $(mktemp -d) + +set -x + +rm -rf /etc/yum.repos.d/* +cat > /etc/yum.repos.d/vmcheck.repo << EOF +[test-repo] +name=test-repo +baseurl=file:///${KOLA_EXT_DATA}/rpm-repos/0 +gpgcheck=0 +EOF + +if rpm-ostree install foomodular 2>err.txt; then + assert_not_reached "successfully installed a modular pkg?" +fi +assert_file_has_content_literal err.txt "Packages not found: foomodular" +echo "ok can't install modular pkg by default" + +if rpm-ostree module install foomodular 2>err.txt; then + assert_not_reached "successfully installed a module with multiple streams without a default stream" +fi +assert_file_has_content err.txt "Cannot enable more streams .* at the same time" +echo "ok can't install module with multiple streams" + +if rpm-ostree module enable foomodular 2>err.txt; then + assert_not_reached "successfully enabled a module with multiple streams without a default stream" +fi +assert_file_has_content err.txt "Cannot enable more streams .* at the same time" +echo "ok can't enable module with multiple streams" + +rpm-ostree module enable foomodular:no-profile +rpm-ostree install foomodular +rpm-ostree status > output.txt +assert_file_has_content_literal output.txt "EnabledModules: foomodular:no-profile" +assert_file_has_content_literal output.txt "LayeredPackages: foomodular" +rpm-ostree cleanup -p +echo "ok enable module" + +if rpm-ostree module install foomodular:no-default-profile 2>err.txt; then + assert_not_reached "successfully installed a module without default profile?" +fi +assert_file_has_content_literal err.txt "No default profile found" +echo "ok can't install module without default profile" + +rpm-ostree module install foomodular:no-default-profile/myprof +rpm-ostree status > output.txt +assert_file_has_content_literal output.txt "LayeredModules: foomodular:no-default-profile/myprof" +rpm-ostree cleanup -p +echo "ok install module without default profile" + +rpm-ostree module install foomodular:with-default-profile +rpm-ostree status > output.txt +assert_file_has_content_literal output.txt "LayeredModules: foomodular:with-default-profile" +rpm-ostree cleanup -p +echo "ok install module with default profile" diff --git a/tests/kolainst/kolainst-build.sh b/tests/kolainst/kolainst-build.sh index 5ba4abbbb1..ddb1845183 100755 --- a/tests/kolainst/kolainst-build.sh +++ b/tests/kolainst/kolainst-build.sh @@ -56,6 +56,23 @@ build_rpm testdaemon \ # Will be useful for testing cancellation build_rpm testpkg-post-infinite-loop \ post "echo entering testpkg-post-infinite-loop 1>&2; while true; do sleep 1h; done" +# Test module +build_rpm foomodular +build_module foomodular \ + stream no-profile \ + rpm foomodular-0:1.0-1.x86_64 +build_module foomodular \ + stream no-default-profile \ + profile myprof:foomodular \ + profile myotherprof:foomodular \ + rpm foomodular-0:1.0-1.x86_64 +build_module foomodular \ + stream with-default-profile \ + profile default:foomodular \ + profile myotherprof:foomodular \ + rpm foomodular-0:1.0-1.x86_64 +build_module_defaults foomodular \ + defprofile with-default-profile:default mv ${test_tmpdir}/yumrepo/* ${test_tmpdir}/rpm-repos/${repover}