From 2e9ecbbba290b2988f0ce031147d49dab1603c17 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 5 Mar 2022 09:38:25 -0500 Subject: [PATCH] wip --- Cargo.lock | 1 - Cargo.toml | 4 ++-- rust/src/composepost.rs | 2 +- rust/src/container.rs | 36 ++++++++++++++++++++++++++++----- rust/src/lib.rs | 1 + src/libpriv/rpmostree-refts.cxx | 3 +++ src/libpriv/rpmostree-refts.h | 2 ++ 7 files changed, 40 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96aa6c624a..671749691c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1612,7 +1612,6 @@ dependencies = [ [[package]] name = "ostree-ext" version = "0.6.5" -source = "git+https://github.com/cgwalters/ostree-rs-ext?branch=tar-split#59068f516792e82f7e7a3d434995056be159bfdb" dependencies = [ "anyhow", "async-compression", diff --git a/Cargo.toml b/Cargo.toml index 61ba20e4d3..a47151d70b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,5 +111,5 @@ sanitizers = [] default = [] [patch.crates-io] -ostree-ext = { git = "https://github.com/cgwalters/ostree-rs-ext", branch = "tar-split" } -#ostree-ext = { path = "../../ostreedev/ostree-rs-ext/lib" } +#ostree-ext = { git = "https://github.com/cgwalters/ostree-rs-ext", branch = "tar-split" } +ostree-ext = { path = "../../ostreedev/ostree-rs-ext/lib" } diff --git a/rust/src/composepost.rs b/rust/src/composepost.rs index 5ee58b0eb0..2e59b17da1 100644 --- a/rust/src/composepost.rs +++ b/rust/src/composepost.rs @@ -43,7 +43,7 @@ pub(crate) static COMPAT_VARLIB_SYMLINKS: &[&str] = &["alternatives", "vagrant"] /* See rpmostree-core.h */ const RPMOSTREE_BASE_RPMDB: &str = "usr/lib/sysimage/rpm-ostree-base-db"; -const RPMOSTREE_RPMDB_LOCATION: &str = "usr/share/rpm"; +pub(crate) const RPMOSTREE_RPMDB_LOCATION: &str = "usr/share/rpm"; const RPMOSTREE_SYSIMAGE_RPMDB: &str = "usr/lib/sysimage/rpm"; pub(crate) const TRADITIONAL_RPMDB_LOCATION: &str = "var/lib/rpm"; diff --git a/rust/src/container.rs b/rust/src/container.rs index 66b86d65ab..932d68a8de 100644 --- a/rust/src/container.rs +++ b/rust/src/container.rs @@ -8,9 +8,12 @@ use std::rc::Rc; use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; +use cap_std::fs::Dir; +use cap_std_ext::cap_std; +use cap_std_ext::prelude::*; use chrono::prelude::*; use ostree::glib; -use ostree_ext::chunking::Chunking; +use ostree_ext::chunking::{Chunking, ObjectMetaSized}; use ostree_ext::objectsource::{ ContentID, ObjectMeta, ObjectMetaMap, ObjectMetaSet, ObjectSourceMeta, }; @@ -38,6 +41,10 @@ struct ContentMappingOpts { #[structopt(long = "ref")] ostree_ref: String, + + #[structopt(long)] + /// Output content metadata as JSON + write_contentmeta_json: Option, } #[derive(Debug)] @@ -210,6 +217,7 @@ pub fn content_mapping(args: &[&str]) -> Result<()> { state.packagemeta.insert(ObjectSourceMeta { identifier: Rc::clone(&state.unpackaged_id), name: Rc::clone(&state.unpackaged_id), + srcid: Rc::clone(&state.unpackaged_id), // Assume that content in here changes frequently. change_time_offset: u32::MAX, }); @@ -254,6 +262,7 @@ pub fn content_mapping(args: &[&str]) -> Result<()> { state.packagemeta.insert(ObjectSourceMeta { identifier: Rc::clone(&nevra), name: Rc::clone(&nevra), + srcid: Rc::from(pkgmeta.src_pkg().to_str().unwrap()), change_time_offset, }); } @@ -271,21 +280,29 @@ pub fn content_mapping(args: &[&str]) -> Result<()> { state.content.insert(checksum.to_string(), Rc::clone(&name)); state.packagemeta.insert(ObjectSourceMeta { identifier: Rc::clone(&name), - name, + name: Rc::clone(&name), + srcid: Rc::clone(&name), change_time_offset: u32::MAX, }); state.skip.insert(path); } } + let rpmdb = root.resolve_relative_path(crate::composepost::RPMOSTREE_RPMDB_LOCATION); + if rpmdb.query_exists(gio::NONE_CANCELLABLE) { + // TODO add mapping for rpmdb + } + // Walk the filesystem build_mapping_recurse(&mut Utf8PathBuf::from("/"), &root, &q, &mut state)?; + let src_pkgs: HashSet<_> = state.packagemeta.iter().map(|p| &p.srcid).collect(); // Print out information about what we found println!( - "{} objects in {} packages", + "{} objects in {} packages ({} source)", state.content.len(), - state.packagemeta.len() + state.packagemeta.len(), + src_pkgs.len(), ); println!("rpm size: {}", state.rpmsize); println!( @@ -305,8 +322,17 @@ pub fn content_mapping(args: &[&str]) -> Result<()> { // Convert our build state into the state that ostree consumes, discarding // transient data such as the cases of files owned by multiple packages. let meta: ObjectMeta = state.into(); + // Now generate the sized version + let meta = ObjectMetaSized::compute_sizes(&repo, meta)?; + if let Some(v) = opt.write_contentmeta_json { + let v = v.strip_prefix("/").unwrap_or(&v); + let root = Dir::open_ambient_dir("/", cap_std::ambient_authority())?; + root.replace_file_with(v, |w| { + serde_json::to_writer(w, &meta.sizes).map_err(anyhow::Error::msg) + })?; + } // Ask ostree to convert this data into chunks - let chunking = Chunking::from_mapping(&repo, rev.as_str(), &meta)?; + let chunking = Chunking::from_mapping(&repo, rev.as_str(), meta)?; // Just print it for now. chunking.print(); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 396f3408de..dd96d08fcc 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -695,6 +695,7 @@ pub mod ffi { // Methods on PackageMeta fn size(self: &PackageMeta) -> u64; fn buildtime(self: &PackageMeta) -> u64; + fn src_pkg(self: &PackageMeta) -> &CxxString; } // rpmostree-package-variants.h diff --git a/src/libpriv/rpmostree-refts.cxx b/src/libpriv/rpmostree-refts.cxx index f1458e6f16..cb16c4466c 100644 --- a/src/libpriv/rpmostree-refts.cxx +++ b/src/libpriv/rpmostree-refts.cxx @@ -20,6 +20,8 @@ #include "config.h" +#include +#include #include #include "rpmostree-refts.h" #include "rpmostree-rpm-util.h" @@ -110,6 +112,7 @@ RpmTs::package_meta(const rust::Str name) const { previous = util::move_nullify(nevra); retval->_size = headerGetNumber(h, RPMTAG_LONGARCHIVESIZE); retval->_buildtime = headerGetNumber(h, RPMTAG_BUILDTIME); + retval->_src_pkg = headerGetString(h, RPMTAG_SOURCERPM); } else { diff --git a/src/libpriv/rpmostree-refts.h b/src/libpriv/rpmostree-refts.h index 92152eb924..d5c7ccdecf 100644 --- a/src/libpriv/rpmostree-refts.h +++ b/src/libpriv/rpmostree-refts.h @@ -53,9 +53,11 @@ namespace rpmostreecxx { struct PackageMeta { uint64_t _size; uint64_t _buildtime; + std::string _src_pkg; uint64_t size() const { return _size; }; uint64_t buildtime() const { return _buildtime; }; + const std::string& src_pkg() const { return _src_pkg; }; }; // A simple C++ wrapper for a librpm C type, so we can expose it to Rust via cxx.rs.