Skip to content

Commit

Permalink
WIP: Add support for modules
Browse files Browse the repository at this point in the history
Add support for composing with modules and for package layering modules
on the client-side.

This doesn't support modules in `rpm-ostree compose extensions` yet,
which is relevant for RHCOS. We can look at adding that in a follow-up.
  • Loading branch information
jlebon committed May 7, 2021
1 parent c7bf087 commit ce20364
Show file tree
Hide file tree
Showing 23 changed files with 648 additions and 34 deletions.
9 changes: 9 additions & 0 deletions docs/treefile.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,15 @@ It supports the following parameters:
* `packages`: Array of strings, required: List of packages to install.
* `repo`: String, required: Name of the repo from which to fetch packages.

* `modules`: Object, optional: Describes RPM modules to enable or install. Two
keys are supported:
* `enable`: Array of strings, required: Set of RPM module specs to enable
(the same formats as dnf are supported, e.g. `NAME[:STREAM]`).
One can then cherry-pick specific packages from the enabled modules via
`packages`.
* `install`: Array of strings, required: Set of RPM module specs to install
(the same formats as dnf are supported, e.g. `NAME[:STREAM][/PROFILE]`).

* `ostree-layers`: Array of strings, optional: After all packages are unpacked,
check out these OSTree refs, which must already be in the destination repository.
Any conflicts with packages will be an error.
Expand Down
2 changes: 1 addition & 1 deletion libdnf
1 change: 1 addition & 0 deletions rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,7 @@ mod lockfile;
pub(crate) use self::lockfile::*;
mod live;
pub(crate) use self::live::*;
pub mod modularity;
mod nameservice;
// An origin parser in Rust but only built when testing until
// we're ready to try porting the C++ code.
Expand Down
1 change: 1 addition & 0 deletions rust/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ fn inner_main(args: &Vec<&str>) -> Result<()> {
match args.get(1).map(|s| *s) {
// Add custom Rust commands here, and also in `libmain.cxx` if user-visible.
Some("countme") => rpmostree_rust::countme::entrypoint(args),
Some("module") => rpmostree_rust::modularity::entrypoint(args),
_ => {
// Otherwise fall through to C++ main().
Ok(rpmostree_rust::ffi::rpmostree_main(&args)?)
Expand Down
128 changes: 128 additions & 0 deletions rust/src/modularity.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
//! Implementation of the client-side of "rpm-ostree module".

// SPDX-License-Identifier: Apache-2.0 OR MIT

use anyhow::{anyhow, bail, Result};
use gio::DBusProxyExt;
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "rpm-ostree module", no_version)]
#[structopt(rename_all = "kebab-case")]
enum Opt {
/// Enable a module
Enable(InstallOpts),
/// Disable a module
Disable(InstallOpts),
/// Install a module
Install(InstallOpts),
/// Uninstall a module
Uninstall(InstallOpts),
}

#[derive(Debug, StructOpt)]
struct InstallOpts {
#[structopt(parse(from_str))]
modules: Vec<String>,
#[structopt(long)]
reboot: bool,
#[structopt(long)]
lock_finalization: bool,
#[structopt(long)]
dry_run: bool,
}

const OPT_KEY_ENABLE_MODULES: &str = "enable-modules";
const OPT_KEY_DISABLE_MODULES: &str = "disable-modules";
const OPT_KEY_INSTALL_MODULES: &str = "install-modules";
const OPT_KEY_UNINSTALL_MODULES: &str = "uninstall-modules";

pub fn entrypoint(args: &[&str]) -> Result<()> {
let opt = Opt::from_iter(args.iter().skip(1));
match opt {
Opt::Enable(ref opts) => enable(opts)?,
Opt::Disable(ref opts) => disable(opts)?,
Opt::Install(ref opts) => install(opts)?,
Opt::Uninstall(ref opts) => uninstall(opts)?,
};
Ok(())
}

// XXX: Should split out a lot of the below into a more generic Rust wrapper around
// UpdateDeployment() like we have on the C side.

fn get_modifiers_variant(key: &str, modules: &[String]) -> Result<glib::Variant> {
let r = glib::VariantDict::new(None);
r.insert_value(key, &crate::variant_utils::new_variant_strv(modules));
Ok(r.end())
}
fn get_options_variant(opts: &InstallOpts) -> Result<glib::Variant> {
let r = glib::VariantDict::new(None);
r.insert("no-pull-base", &true);
r.insert("reboot", &opts.reboot);
r.insert("lock-finalization", &opts.lock_finalization);
r.insert("dry-run", &opts.dry_run);
Ok(r.end())
}

fn enable(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_ENABLE_MODULES, opts)
}

fn disable(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_DISABLE_MODULES, opts)
}

fn install(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_INSTALL_MODULES, opts)
}

fn uninstall(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_UNINSTALL_MODULES, opts)
}

fn modules_impl(key: &str, opts: &InstallOpts) -> Result<()> {
if opts.modules.is_empty() {
bail!("At least one module must be specified");
}

let client = &mut crate::client::ClientConnection::new()?;
let previous_deployment = client
.get_os_proxy()
.get_cached_property("DefaultDeployment")
.ok_or_else(|| anyhow!("Failed to find default-deployment property"))?;
let modifiers = get_modifiers_variant(key, &opts.modules)?;
let options = get_options_variant(opts)?;
let params = crate::variant_utils::new_variant_tuple(&[modifiers, options]);
let reply = &client.get_os_proxy().call_sync(
"UpdateDeployment",
Some(&params),
gio::DBusCallFlags::NONE,
-1,
gio::NONE_CANCELLABLE,
)?;
let reply_child = crate::variant_utils::variant_tuple_get(reply, 0)
.ok_or_else(|| anyhow!("Invalid reply"))?;
let txn_address = reply_child
.get_str()
.ok_or_else(|| anyhow!("Expected string transaction address"))?;
client.transaction_connect_progress_sync(txn_address)?;
let new_deployment = client
.get_os_proxy()
.get_cached_property("DefaultDeployment")
.ok_or_else(|| anyhow!("Failed to find default-deployment property"))?;
if previous_deployment != new_deployment {
unsafe {
crate::ffi::print_treepkg_diff_from_sysroot_path(
"/",
crate::ffi::RpmOstreeDiffPrintFormat::RPMOSTREE_DIFF_PRINT_FORMAT_FULL_MULTILINE,
0,
std::ptr::null_mut(),
);
}
}
if opts.dry_run {
println!("Exiting because of '--dry-run' option");
}
Ok(())
}
58 changes: 58 additions & 0 deletions rust/src/treefile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,17 @@ fn treefile_parse_stream<R: io::Read>(
}
}

// to be consistent, we also support whitespace-separated modules
if let Some(mut modules) = treefile.modules.take() {
if let Some(enable) = modules.enable.take() {
modules.enable = Some(whitespace_split_packages(&enable)?);
}
if let Some(install) = modules.install.take() {
modules.install = Some(whitespace_split_packages(&install)?);
}
treefile.modules = Some(modules);
}

if let Some(repo_packages) = treefile.repo_packages.take() {
treefile.repo_packages = Some(
repo_packages
Expand Down Expand Up @@ -320,6 +331,18 @@ fn merge_hashset_field<T: Eq + std::hash::Hash>(
}
}

/// Merge modules fields.
fn merge_modules(dest: &mut Option<ModulesConfig>, src: &mut Option<ModulesConfig>) {
if let Some(mut srcv) = src.take() {
if let Some(mut destv) = dest.take() {
merge_vec_field(&mut destv.enable, &mut srcv.enable);
merge_vec_field(&mut destv.install, &mut srcv.install);
srcv = destv;
}
*dest = Some(srcv);
}
}

/// Given two configs, merge them.
fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
macro_rules! merge_basics {
Expand Down Expand Up @@ -389,6 +412,8 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
remove_from_packages,
repo_packages
);

merge_modules(&mut dest.modules, &mut src.modules);
}

/// Merge the treefile externals. There are currently only two keys that
Expand Down Expand Up @@ -1094,6 +1119,8 @@ pub(crate) struct TreeComposeConfig {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(rename = "repo-packages")]
pub(crate) repo_packages: Option<Vec<RepoPackage>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) modules: Option<ModulesConfig>,
// Deprecated option
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) bootstrap_packages: Option<Vec<String>>,
Expand Down Expand Up @@ -1218,6 +1245,14 @@ pub(crate) struct RepoPackage {
pub(crate) packages: Vec<String>,
}

#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
pub(crate) struct ModulesConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) enable: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) install: Option<Vec<String>>,
}

#[derive(Serialize, Deserialize, Debug, Default)]
pub(crate) struct LegacyTreeComposeConfigFields {
#[serde(skip_serializing)]
Expand Down Expand Up @@ -1351,6 +1386,12 @@ pub(crate) mod tests {
- repo: baserepo
packages:
- blah bloo
modules:
enable:
- foobar:2.0
install:
- nodejs:15
- swig:3.0/complete sway:rolling
"#};

// This one has "comments" (hence unknown keys)
Expand Down Expand Up @@ -1616,6 +1657,11 @@ pub(crate) mod tests {
- repo: foo2
packages:
- qwert
modules:
enable:
- dodo
install:
- bazboo
"},
)?;
let mut buf = VALID_PRELUDE.to_string();
Expand All @@ -1637,6 +1683,18 @@ pub(crate) mod tests {
}
])
);
assert_eq!(
tf.parsed.modules,
Some(ModulesConfig {
enable: Some(vec!["dodo".into(), "foobar:2.0".into()]),
install: Some(vec![
"bazboo".into(),
"nodejs:15".into(),
"swig:3.0/complete".into(),
"sway:rolling".into(),
])
},)
);
Ok(())
}

Expand Down
6 changes: 6 additions & 0 deletions rust/src/variant_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
use std::borrow::Cow;

use glib::translate::*;
use glib::ToVariant;

// These constants should really be in gtk-rs
lazy_static::lazy_static! {
Expand Down Expand Up @@ -51,6 +52,11 @@ pub(crate) fn new_variant_array(ty: &glib::VariantTy, children: &[glib::Variant]
}
}

pub(crate) fn new_variant_strv(strv: &[impl AsRef<str>]) -> glib::Variant {
let v: Vec<glib::Variant> = strv.iter().map(|s| s.as_ref().to_variant()).collect();
new_variant_array(&TY_S, &v)
}

pub(crate) fn is_container(v: &glib::Variant) -> bool {
unsafe { glib_sys::g_variant_is_container(v.to_glib_none().0) != glib_sys::GFALSE }
}
Expand Down
6 changes: 2 additions & 4 deletions src/app/libmain.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,8 @@ static RpmOstreeCommand commands[] = {
/* Rust-implemented commands; they're here so that they show up in `rpm-ostree
* --help` alongside the other commands, but the command itself is fully
* handled Rust side. */
/*
{ "my-rust-command", static_cast<RpmOstreeBuiltinFlags>(0),
"Cool thing my command does", NULL },
*/
{ "module", static_cast<RpmOstreeBuiltinFlags>(0),
"Commands to install/uninstall modules", NULL },
/* Legacy aliases */
{ "pkg-add", static_cast<RpmOstreeBuiltinFlags>(RPM_OSTREE_BUILTIN_FLAG_HIDDEN),
NULL, rpmostree_builtin_install },
Expand Down
19 changes: 19 additions & 0 deletions src/app/rpmostree-builtin-status.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,10 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy,

const gchar *origin_refspec;
g_autofree const gchar **origin_packages = NULL;
g_autofree const gchar **origin_modules = NULL;
g_autofree const gchar **origin_modules_enabled = NULL;
g_autofree const gchar **origin_requested_packages = NULL;
g_autofree const gchar **origin_requested_modules = NULL;
g_autofree const gchar **origin_requested_local_packages = NULL;
g_autoptr(GVariant) origin_base_removals = NULL;
g_autofree const gchar **origin_requested_base_removals = NULL;
Expand All @@ -583,8 +586,14 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy,
{
origin_packages =
lookup_array_and_canonicalize (dict, "packages");
origin_modules =
lookup_array_and_canonicalize (dict, "modules");
origin_modules_enabled =
lookup_array_and_canonicalize (dict, "modules-enabled");
origin_requested_packages =
lookup_array_and_canonicalize (dict, "requested-packages");
origin_requested_modules =
lookup_array_and_canonicalize (dict, "requested-modules");
origin_requested_local_packages =
lookup_array_and_canonicalize (dict, "requested-local-packages");
origin_base_removals =
Expand Down Expand Up @@ -966,10 +975,20 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy,
/* requested-packages - packages = inactive (i.e. dormant requests) */
print_packages ("InactiveRequests", max_key_len,
origin_requested_packages, origin_packages);
if (origin_requested_modules && opt_verbose)
/* requested-modules - modules = inactive (i.e. dormant requests) */
print_packages ("InactiveModuleRequests", max_key_len,
origin_requested_modules, origin_modules);

if (origin_packages)
print_packages ("LayeredPackages", max_key_len,
origin_packages, NULL);
if (origin_modules)
print_packages ("LayeredModules", max_key_len,
origin_modules, NULL);
if (origin_modules_enabled)
print_packages ("EnabledModules", max_key_len,
origin_modules_enabled, NULL);

if (origin_requested_local_packages)
print_packages ("LocalPackages", max_key_len,
Expand Down
15 changes: 15 additions & 0 deletions src/app/rpmostree-compose-builtin-tree.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,12 @@ impl_install_tree (RpmOstreeTreeComposeContext *self,
return glnx_throw_errno_prefix (error, "fchdir");
}

/* We don't support installing modules in non-unified mode, because it relies
* on the core writing metadata to the commit metadata (obviously this could
* be supported, but meh...) */
if (!opt_unified_core && json_object_has_member (self->treefile, "modules"))
return glnx_throw (error, "Composing with modules requires --unified-core");

/* Read the previous commit. Note we don't actually *need* the full commit; really, only
* if one uses `check-passwd: { "type": "previous" }`. There are a few other optimizations
* too, e.g. using the previous SELinux policy in unified core. Also, we might need the
Expand Down Expand Up @@ -953,6 +959,15 @@ impl_install_tree (RpmOstreeTreeComposeContext *self,
if (!inject_advisories (self, cancellable, error))
return FALSE;

/* embed modules layered in fully resolved NSVCA/P form */
GHashTable *modules = rpmostree_context_get_modules_installed (self->corectx);
if (modules && g_hash_table_size (modules) > 0)
{
g_autofree char** strv = (char**)g_hash_table_get_keys_as_array (modules, NULL);
g_hash_table_insert (self->metadata, g_strdup ("rpmostree.modules.nsvcap"),
g_variant_new_strv (strv, -1));
}

/* Destroy this now so the libdnf stack won't have any references
* into the filesystem before we manipulate it.
*/
Expand Down
Loading

0 comments on commit ce20364

Please sign in to comment.