Skip to content

Commit

Permalink
Add support for modules
Browse files Browse the repository at this point in the history
Add support for composing with modules and for package layering modules
on the client-side.

This doesn't support modules in `rpm-ostree compose extensions` yet,
which is relevant for RHCOS. We can look at adding that in a follow-up.
  • Loading branch information
jlebon committed Jul 23, 2021
1 parent ff4be57 commit e324b40
Show file tree
Hide file tree
Showing 27 changed files with 844 additions and 38 deletions.
9 changes: 9 additions & 0 deletions docs/treefile.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,15 @@ It supports the following parameters:
* `packages`: Array of strings, required: List of packages to install.
* `repo`: String, required: Name of the repo from which to fetch packages.

* `modules`: Object, optional: Describes RPM modules to enable or install. Two
keys are supported:
* `enable`: Array of strings, required: Set of RPM module specs to enable
(the same formats as dnf are supported, e.g. `NAME[:STREAM]`).
One can then cherry-pick specific packages from the enabled modules via
`packages`.
* `install`: Array of strings, required: Set of RPM module specs to install
(the same formats as dnf are supported, e.g. `NAME[:STREAM][/PROFILE]`).

* `ostree-layers`: Array of strings, optional: After all packages are unpacked,
check out these OSTree refs, which must already be in the destination repository.
Any conflicts with packages will be an error.
Expand Down
10 changes: 10 additions & 0 deletions rust/src/daemon.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@ fn deployment_populate_variant_origin(

// Package mappings. Note these are inserted unconditionally, even if empty.
vdict_insert_optvec(&dict, "requested-packages", tf.packages.as_ref());
vdict_insert_optvec(
&dict,
"requested-modules",
tf.modules.as_ref().map(|m| m.install.as_ref()).flatten(),
);
vdict_insert_optvec(
&dict,
"modules-enabled",
tf.modules.as_ref().map(|m| m.enable.as_ref()).flatten(),
);
vdict_insert_optmap(
&dict,
"requested-local-packages",
Expand Down
3 changes: 3 additions & 0 deletions rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,8 @@ pub mod ffi {
fn get_packages_local(&self) -> Vec<String>;
fn get_packages_override_replace_local(&self) -> Vec<String>;
fn get_packages_override_remove(&self) -> Vec<String>;
fn get_modules_enable(&self) -> Vec<String>;
fn get_modules_install(&self) -> Vec<String>;
fn get_exclude_packages(&self) -> Vec<String>;
fn get_install_langs(&self) -> Vec<String>;
fn format_install_langs_macro(&self) -> String;
Expand Down Expand Up @@ -618,6 +620,7 @@ mod lockfile;
pub(crate) use self::lockfile::*;
mod live;
pub(crate) use self::live::*;
pub mod modularity;
mod nameservice;
mod origin;
pub(crate) use self::origin::*;
Expand Down
1 change: 1 addition & 0 deletions rust/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ fn inner_main() -> Result<i32> {
Some("countme") => rpmostree_rust::countme::entrypoint(&args).map(|_| 0),
Some("cliwrap") => rpmostree_rust::cliwrap::entrypoint(&args).map(|_| 0),
Some("ex-container") => rpmostree_rust::container::entrypoint(&args).map(|_| 0),
Some("module") => rpmostree_rust::modularity::entrypoint(&args).map(|_| 0),
_ => {
// Otherwise fall through to C++ main().
Ok(rpmostree_rust::ffi::rpmostree_main(&args)?)
Expand Down
130 changes: 130 additions & 0 deletions rust/src/modularity.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
//! Implementation of the client-side of "rpm-ostree module".

// SPDX-License-Identifier: Apache-2.0 OR MIT

use anyhow::{anyhow, bail, Result};
use gio::DBusProxyExt;
use ostree_ext::variant_utils;
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "rpm-ostree module", no_version)]
#[structopt(rename_all = "kebab-case")]
enum Opt {
/// Enable a module
Enable(InstallOpts),
/// Disable a module
Disable(InstallOpts),
/// Install a module
Install(InstallOpts),
/// Uninstall a module
Uninstall(InstallOpts),
}

#[derive(Debug, StructOpt)]
struct InstallOpts {
#[structopt(parse(from_str))]
modules: Vec<String>,
#[structopt(long)]
reboot: bool,
#[structopt(long)]
lock_finalization: bool,
#[structopt(long)]
dry_run: bool,
}

const OPT_KEY_ENABLE_MODULES: &str = "enable-modules";
const OPT_KEY_DISABLE_MODULES: &str = "disable-modules";
const OPT_KEY_INSTALL_MODULES: &str = "install-modules";
const OPT_KEY_UNINSTALL_MODULES: &str = "uninstall-modules";

pub fn entrypoint(args: &[&str]) -> Result<()> {
match Opt::from_iter(args.iter().skip(1)) {
Opt::Enable(ref opts) => enable(opts),
Opt::Disable(ref opts) => disable(opts),
Opt::Install(ref opts) => install(opts),
Opt::Uninstall(ref opts) => uninstall(opts),
}
}

// XXX: Should split out a lot of the below into a more generic Rust wrapper around
// UpdateDeployment() like we have on the C side.

fn get_modifiers_variant(key: &str, modules: &[String]) -> Result<glib::Variant> {
let r = glib::VariantDict::new(None);
r.insert_value(key, &crate::variant_utils::new_variant_strv(modules));
Ok(r.end())
}

fn get_options_variant(opts: &InstallOpts) -> Result<glib::Variant> {
let r = glib::VariantDict::new(None);
r.insert("no-pull-base", &true);
r.insert("reboot", &opts.reboot);
r.insert("lock-finalization", &opts.lock_finalization);
r.insert("dry-run", &opts.dry_run);
Ok(r.end())
}

fn enable(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_ENABLE_MODULES, opts)
}

fn disable(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_DISABLE_MODULES, opts)
}

fn install(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_INSTALL_MODULES, opts)
}

fn uninstall(opts: &InstallOpts) -> Result<()> {
modules_impl(OPT_KEY_UNINSTALL_MODULES, opts)
}

fn modules_impl(key: &str, opts: &InstallOpts) -> Result<()> {
eprintln!("NOTE: Modularity support is experimental and subject to change.");

if opts.modules.is_empty() {
bail!("At least one module must be specified");
}

let client = &mut crate::client::ClientConnection::new()?;
let previous_deployment = client
.get_os_proxy()
.get_cached_property("DefaultDeployment")
.ok_or_else(|| anyhow!("Failed to find default-deployment property"))?;
let modifiers = get_modifiers_variant(key, &opts.modules)?;
let options = get_options_variant(opts)?;
let params = variant_utils::new_variant_tuple(&[modifiers, options]);
let reply = &client.get_os_proxy().call_sync(
"UpdateDeployment",
Some(&params),
gio::DBusCallFlags::NONE,
-1,
gio::NONE_CANCELLABLE,
)?;
let reply_child = crate::variant_utils::variant_tuple_get(reply, 0)
.ok_or_else(|| anyhow!("Invalid reply"))?;
let txn_address = reply_child
.get_str()
.ok_or_else(|| anyhow!("Expected string transaction address"))?;
client.transaction_connect_progress_sync(txn_address)?;
let new_deployment = client
.get_os_proxy()
.get_cached_property("DefaultDeployment")
.ok_or_else(|| anyhow!("Failed to find default-deployment property"))?;
if previous_deployment != new_deployment {
unsafe {
crate::ffi::print_treepkg_diff_from_sysroot_path(
"/",
crate::ffi::RpmOstreeDiffPrintFormat::RPMOSTREE_DIFF_PRINT_FORMAT_FULL_MULTILINE,
0,
std::ptr::null_mut(),
);
}
}
if opts.dry_run {
println!("Exiting because of '--dry-run' option");
}
Ok(())
}
30 changes: 30 additions & 0 deletions rust/src/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use std::result::Result as StdResult;
const ORIGIN: &str = "origin";
const RPMOSTREE: &str = "rpmostree";
const PACKAGES: &str = "packages";
const MODULES: &str = "modules";
const OVERRIDES: &str = "overrides";

/// The set of keys that we parse as BTreeMap and need to ignore ordering changes.
Expand All @@ -43,6 +44,14 @@ pub(crate) fn origin_to_treefile_inner(kf: &KeyFile) -> Result<Box<Treefile>> {
cfg.derive.base_refspec = Some(refspec_str);
cfg.packages = parse_stringlist(&kf, PACKAGES, "requested")?;
cfg.derive.packages_local = parse_localpkglist(&kf, PACKAGES, "requested-local")?;
let modules_enable = parse_stringlist(&kf, MODULES, "enable")?;
let modules_install = parse_stringlist(&kf, MODULES, "install")?;
if modules_enable.is_some() || modules_install.is_some() {
cfg.modules = Some(crate::treefile::ModulesConfig {
enable: modules_enable,
install: modules_install,
});
}
cfg.derive.override_remove = parse_stringlist(&kf, OVERRIDES, "remove")?;
cfg.derive.override_replace_local = parse_localpkglist(&kf, OVERRIDES, "replace-local")?;

Expand Down Expand Up @@ -143,6 +152,16 @@ fn treefile_to_origin_inner(tf: &Treefile) -> Result<glib::KeyFile> {
if let Some(pkgs) = tf.derive.override_replace_local.as_ref() {
set_sha256_nevra_pkgs(&kf, OVERRIDES, "replace-local", pkgs)
}
if let Some(ref modcfg) = tf.modules {
if let Some(modules) = modcfg.enable.as_deref() {
let modules = modules.iter().map(|s| s.as_str());
kf_set_string_list(&kf, MODULES, "enable", modules)
}
if let Some(modules) = modcfg.install.as_deref() {
let modules = modules.iter().map(|s| s.as_str());
kf_set_string_list(&kf, MODULES, "install", modules)
}
}

// Initramfs bits
if let Some(initramfs) = tf.derive.initramfs.as_ref() {
Expand Down Expand Up @@ -332,6 +351,10 @@ pub(crate) mod test {
requested=libvirt;fish;
requested-local=4ed748ba060fce4571e7ef19f3f5ed6209f67dbac8327af0d38ea70b96d2f723:foo-1.2-3.x86_64;
[modules]
enable=foo:2.0;bar:rolling;
install=baz:next/development;
[overrides]
remove=docker;
replace-local=0c7072500af2758e7dc7d7700fed82c3c5f4da7453b4d416e79f75384eee96b0:rpm-ostree-devel-2021.1-2.fc33.x86_64;648ab3ff4d4b708ea180269297de5fa3e972f4481d47b7879c6329272e474d68:rpm-ostree-2021.1-2.fc33.x86_64;8b29b78d0ade6ec3aedb8e3846f036f6f28afe64635d83cb6a034f1004607678:rpm-ostree-libs-2021.1-2.fc33.x86_64;
Expand Down Expand Up @@ -393,6 +416,13 @@ pub(crate) mod test {
tf.parsed.derive.override_commit.unwrap(),
"41af286dc0b172ed2f1ca934fd2278de4a1192302ffa07087cea2682e7d372e3"
);
assert_eq!(
tf.parsed.modules,
Some(crate::treefile::ModulesConfig {
enable: Some(vec!["foo:2.0".into(), "bar:rolling".into()]),
install: Some(vec!["baz:next/development".into()]),
})
);
Ok(())
}

Expand Down
81 changes: 81 additions & 0 deletions rust/src/treefile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,17 @@ fn treefile_parse_stream<R: io::Read>(
}
}

// to be consistent, we also support whitespace-separated modules
if let Some(mut modules) = treefile.modules.take() {
if let Some(enable) = modules.enable.take() {
modules.enable = Some(whitespace_split_packages(&enable)?);
}
if let Some(install) = modules.install.take() {
modules.install = Some(whitespace_split_packages(&install)?);
}
treefile.modules = Some(modules);
}

if let Some(repo_packages) = treefile.repo_packages.take() {
treefile.repo_packages = Some(
repo_packages
Expand Down Expand Up @@ -313,6 +324,18 @@ fn merge_hashset_field<T: Eq + std::hash::Hash>(
}
}

/// Merge modules fields.
fn merge_modules(dest: &mut Option<ModulesConfig>, src: &mut Option<ModulesConfig>) {
if let Some(mut srcv) = src.take() {
if let Some(mut destv) = dest.take() {
merge_vec_field(&mut destv.enable, &mut srcv.enable);
merge_vec_field(&mut destv.install, &mut srcv.install);
srcv = destv;
}
*dest = Some(srcv);
}
}

/// Given two configs, merge them.
fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
macro_rules! merge_basics {
Expand Down Expand Up @@ -384,6 +407,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
);

merge_basic_field(&mut dest.derive.base_refspec, &mut src.derive.base_refspec);
merge_modules(&mut dest.modules, &mut src.modules);
}

/// Merge the treefile externals. There are currently only two keys that
Expand Down Expand Up @@ -595,6 +619,30 @@ impl Treefile {
.collect()
}

pub(crate) fn get_modules_enable(&self) -> Vec<String> {
self.parsed
.modules
.as_ref()
.map(|m| m.enable.as_ref())
.flatten()
.map(|v| v.clone())
.into_iter()
.flatten()
.collect()
}

pub(crate) fn get_modules_install(&self) -> Vec<String> {
self.parsed
.modules
.as_ref()
.map(|m| m.install.as_ref())
.flatten()
.map(|v| v.clone())
.into_iter()
.flatten()
.collect()
}

pub(crate) fn get_packages_override_remove(&self) -> Vec<String> {
self.parsed
.derive
Expand Down Expand Up @@ -1097,6 +1145,8 @@ pub(crate) struct TreeComposeConfig {
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(rename = "repo-packages")]
pub(crate) repo_packages: Option<Vec<RepoPackage>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) modules: Option<ModulesConfig>,
// Deprecated option
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) bootstrap_packages: Option<Vec<String>>,
Expand Down Expand Up @@ -1224,6 +1274,14 @@ pub(crate) struct RepoPackage {
pub(crate) packages: Vec<String>,
}

#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
pub(crate) struct ModulesConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) enable: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) install: Option<Vec<String>>,
}

#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
pub(crate) struct LegacyTreeComposeConfigFields {
#[serde(skip_serializing)]
Expand Down Expand Up @@ -1426,6 +1484,12 @@ pub(crate) mod tests {
- repo: baserepo
packages:
- blah bloo
modules:
enable:
- foobar:2.0
install:
- nodejs:15
- swig:3.0/complete sway:rolling
"#};

// This one has "comments" (hence unknown keys)
Expand Down Expand Up @@ -1720,6 +1784,11 @@ pub(crate) mod tests {
- repo: foo2
packages:
- qwert
modules:
enable:
- dodo
install:
- bazboo
"},
)?;
let mut buf = VALID_PRELUDE.to_string();
Expand All @@ -1741,6 +1810,18 @@ pub(crate) mod tests {
}
])
);
assert_eq!(
tf.parsed.modules,
Some(ModulesConfig {
enable: Some(vec!["dodo".into(), "foobar:2.0".into()]),
install: Some(vec![
"bazboo".into(),
"nodejs:15".into(),
"swig:3.0/complete".into(),
"sway:rolling".into(),
])
},)
);
Ok(())
}

Expand Down
Loading

0 comments on commit e324b40

Please sign in to comment.