WIP: Adapt to and use new ostree-ext tar-split branch

Depends ostreedev/ostree-rs-ext#123

Cargo.toml

@@ -109,3 +109,7 @@ bin-unit-tests = []
 sanitizers = []
 
 default = []
+
+[patch.crates-io]
+ostree-ext = { git = "https://github.com/cgwalters/ostree-rs-ext", branch = "tar-split" }
+#ostree-ext = { path = "../../ostreedev/ostree-rs-ext/lib" }

rust/src/container.rs

@@ -2,7 +2,23 @@
 
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::convert::TryInto;
+use std::rc::Rc;
+
 use anyhow::Result;
+use camino::{Utf8Path, Utf8PathBuf};
+use chrono::prelude::*;
+use ostree::glib;
+use ostree_ext::chunking::Chunking;
+use ostree_ext::objectsource::{
+    ContentID, ObjectMeta, ObjectMetaMap, ObjectMetaSet, ObjectSourceMeta,
+};
+use ostree_ext::prelude::*;
+use ostree_ext::{gio, ostree};
+use structopt::StructOpt;
+
+use crate::cxxrsutil::FFIGObjectReWrap;
 
 /// Main entrypoint for container
 pub async fn entrypoint(args: &[&str]) -> Result<i32> {
@@ -14,3 +30,265 @@ pub async fn entrypoint(args: &[&str]) -> Result<i32> {
     ostree_ext::cli::run_from_iter(args).await?;
     Ok(0)
 }
+
+#[derive(Debug, StructOpt)]
+struct ContentMappingOpts {
+    #[structopt(long)]
+    repo: String,
+
+    #[structopt(long = "ref")]
+    ostree_ref: String,
+}
+
+#[derive(Debug)]
+struct MappingBuilder {
+    /// Maps from package ID to metadata
+    packagemeta: ObjectMetaSet,
+    /// Mapping from content object sha256 to package numeric ID
+    content: ObjectMetaMap,
+    /// Mapping from content object sha256 to package numeric ID
+    duplicates: BTreeMap<String, Vec<ContentID>>,
+    multi_provider: Vec<Utf8PathBuf>,
+
+    unpackaged_id: Rc<str>,
+
+    /// Files that were processed before the global tree walk
+    skip: HashSet<Utf8PathBuf>,
+
+    /// Size according to RPM database
+    rpmsize: u64,
+}
+
+impl MappingBuilder {
+    /// For now, we stick everything that isn't a package inside a single "unpackaged" state.
+    /// In the future though if we support e.g. containers in /usr/share/containers or the
+    /// like, this will need to change.
+    const UNPACKAGED_ID: &'static str = "rpmostree-unpackaged-content";
+}
+
+impl From<MappingBuilder> for ObjectMeta {
+    fn from(b: MappingBuilder) -> ObjectMeta {
+        ObjectMeta {
+            map: b.content,
+            set: b.packagemeta,
+        }
+    }
+}
+
+/// Walk over the whole filesystem, and generate mappings from content object checksums
+/// to the package that owns them.  
+///
+/// In the future, we could compute this much more efficiently by walking that
+/// instead.  But this design is currently oriented towards accepting a single ostree
+/// commit as input.
+fn build_mapping_recurse(
+    path: &mut Utf8PathBuf,
+    dir: &gio::File,
+    ts: &crate::ffi::RpmTs,
+    state: &mut MappingBuilder,
+) -> Result<()> {
+    use std::collections::btree_map::Entry;
+    let cancellable = gio::NONE_CANCELLABLE;
+    let e = dir.enumerate_children(
+        "standard::name,standard::type",
+        gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS,
+        cancellable,
+    )?;
+    for child in e {
+        let childi = child?;
+        let name: Utf8PathBuf = childi.name().try_into()?;
+        let child = dir.child(&name);
+        path.push(&name);
+        match childi.file_type() {
+            gio::FileType::Regular | gio::FileType::SymbolicLink => {
+                let child = child.downcast::<ostree::RepoFile>().unwrap();
+
+                // Remove the skipped path, since we can't hit it again.
+                if state.skip.remove(Utf8Path::new(path)) {
+                    path.pop();
+                    continue;
+                }
+
+                let mut pkgs = ts.packages_providing_file(path.as_str())?;
+                // Let's be deterministic (but _unstable because we don't care about behavior of equal strings)
+                pkgs.sort_unstable();
+                // For now, we pick the alphabetically first package providing a file
+                let mut pkgs = pkgs.into_iter();
+                let pkgid = pkgs
+                    .next()
+                    .map(|v| {
+                        // Safety: we should have the package in metadata
+                        let meta = state.packagemeta.get(v.as_str()).unwrap();
+                        Rc::clone(&meta.identifier)
+                    })
+                    .unwrap_or_else(|| Rc::clone(&state.unpackaged_id));
+                // Track cases of duplicate owners
+                match pkgs.len() {
+                    0 => {}
+                    _ => {
+                        state.multi_provider.push(path.clone());
+                    }
+                }
+
+                let checksum = child.checksum().unwrap().to_string();
+                match state.content.entry(checksum) {
+                    Entry::Vacant(v) => {
+                        v.insert(pkgid);
+                    }
+                    Entry::Occupied(_) => {
+                        let checksum = child.checksum().unwrap().to_string();
+                        let v = state.duplicates.entry(checksum).or_default();
+                        v.push(pkgid);
+                    }
+                }
+            }
+            gio::FileType::Directory => {
+                build_mapping_recurse(path, &child, ts, state)?;
+            }
+            o => anyhow::bail!("Unhandled file type: {}", o),
+        }
+        path.pop();
+    }
+    Ok(())
+}
+
+/// Print out information about how we would generate "chunks" (i.e. OCI layers) for this ostree commit.
+pub fn content_mapping(args: &[&str]) -> Result<()> {
+    let args = args.iter().skip(1);
+    let opt = ContentMappingOpts::from_iter(args);
+    let repo = ostree::Repo::new_for_path(opt.repo.as_str());
+    repo.open(gio::NONE_CANCELLABLE)?;
+    let (root, rev) = repo.read_commit(opt.ostree_ref.as_str(), gio::NONE_CANCELLABLE)?;
+    let pkglist = {
+        let repo = repo.gobj_rewrap();
+        let cancellable = gio::Cancellable::new();
+        unsafe {
+            let r = crate::ffi::package_variant_list_for_commit(
+                repo,
+                rev.as_str(),
+                cancellable.gobj_rewrap(),
+            )?;
+            let r: glib::Variant = glib::translate::from_glib_full(r as *mut _);
+            r
+        }
+    };
+
+    // Open the RPM database for this commit.
+    let q = crate::ffi::rpmts_for_commit(repo.gobj_rewrap(), rev.as_str())?;
+
+    let mut state = MappingBuilder {
+        unpackaged_id: Rc::from(MappingBuilder::UNPACKAGED_ID),
+        packagemeta: Default::default(),
+        content: Default::default(),
+        duplicates: Default::default(),
+        multi_provider: Default::default(),
+        skip: Default::default(),
+        rpmsize: Default::default(),
+    };
+    // Insert metadata for unpakaged content.
+    state.packagemeta.insert(ObjectSourceMeta {
+        identifier: Rc::clone(&state.unpackaged_id),
+        name: Rc::clone(&state.unpackaged_id),
+        // Assume that content in here changes frequently.
+        change_time_offset: u32::MAX,
+    });
+
+    let mut lowest_change_time = None;
+    let mut package_meta = HashMap::new();
+    for pkg in pkglist.iter() {
+        let name = pkg.child_value(0);
+        let pkgid = String::from(name.str().unwrap());
+        let pkgmeta = q.package_meta(&pkgid)?;
+        let buildtime = pkgmeta.buildtime();
+        if let Some((lowid, lowtime)) = lowest_change_time.as_mut() {
+            if *lowtime > buildtime {
+                *lowid = pkgid.clone();
+                *lowtime = buildtime;
+            }
+        } else {
+            lowest_change_time = Some((pkgid.clone(), pkgmeta.buildtime()))
+        }
+        state.rpmsize += pkgmeta.size();
+        package_meta.insert(pkgid, pkgmeta);
+    }
+
+    // SAFETY: There must be at least one package.
+    let (lowest_change_name, lowest_change_time) =
+        lowest_change_time.expect("Failed to find any packages");
+    // Walk over the packages, and generate the `packagemeta` mapping, which is basically a subset of
+    // package metadata abstracted for ostree.  Note that right now, the package metadata includes
+    // both a "unique identifer" and a "human readable name", but for rpm-ostree we're just making
+    // those the same thing.
+    for (name, pkgmeta) in package_meta.iter() {
+        let name = Rc::from(name.as_str());
+        let buildtime = pkgmeta.buildtime();
+        let change_time_offset_secs: u32 = buildtime
+            .checked_sub(lowest_change_time)
+            .unwrap()
+            .try_into()
+            .unwrap();
+        // Convert to hours, because there's no strong use for caring about the relative difference of builds in terms
+        // of minutes or seconds.
+        let change_time_offset = change_time_offset_secs / (60 * 60);
+        state.packagemeta.insert(ObjectSourceMeta {
+            identifier: Rc::clone(&name),
+            name: Rc::clone(&name),
+            change_time_offset,
+        });
+    }
+
+    let kernel_dir = ostree_ext::bootabletree::find_kernel_dir(&root, gio::NONE_CANCELLABLE)?;
+    if let Some(kernel_dir) = kernel_dir {
+        let kernel_ver: Utf8PathBuf = kernel_dir.basename().unwrap().try_into()?;
+        let initramfs = kernel_dir.child("initramfs.img");
+        if initramfs.query_exists(gio::NONE_CANCELLABLE) {
+            let path: Utf8PathBuf = initramfs.path().unwrap().try_into()?;
+            let initramfs = initramfs.downcast_ref::<ostree::RepoFile>().unwrap();
+            let checksum = initramfs.checksum().unwrap();
+            let name = format!("initramfs (kernel {})", kernel_ver).into_boxed_str();
+            let name = Rc::from(name);
+            state.content.insert(checksum.to_string(), Rc::clone(&name));
+            state.packagemeta.insert(ObjectSourceMeta {
+                identifier: Rc::clone(&name),
+                name,
+                change_time_offset: u32::MAX,
+            });
+            state.skip.insert(path);
+        }
+    }
+
+    // Walk the filesystem
+    build_mapping_recurse(&mut Utf8PathBuf::from("/"), &root, &q, &mut state)?;
+
+    // Print out information about what we found
+    println!(
+        "{} objects in {} packages",
+        state.content.len(),
+        state.packagemeta.len()
+    );
+    println!("rpm size: {}", state.rpmsize);
+    println!(
+        "Earliest changed package: {} at {}",
+        lowest_change_name,
+        Utc.timestamp_opt(lowest_change_time.try_into().unwrap(), 0)
+            .unwrap()
+    );
+    println!("{} duplicates", state.duplicates.len());
+    if !state.multi_provider.is_empty() {
+        println!("Multiple owners:");
+        for v in state.multi_provider.iter() {
+            println!("  {}", v);
+        }
+    }
+
+    // Convert our build state into the state that ostree consumes, discarding
+    // transient data such as the cases of files owned by multiple packages.
+    let meta: ObjectMeta = state.into();
+    // Ask ostree to convert this data into chunks
+    let chunking = Chunking::from_mapping(&repo, rev.as_str(), &meta)?;
+
+    // Just print it for now.
+    chunking.print();
+
+    Ok(())
+}

rust/src/lib.rs

@@ -674,14 +674,27 @@ pub mod ffi {
 
         fn output_message(msg: &str);
     }
-
     // rpmostree-rpm-util.h
     unsafe extern "C++" {
         include!("rpmostree-rpm-util.h");
+        #[allow(missing_debug_implementations)]
+        type RpmTs;
+        #[allow(missing_debug_implementations)]
+        type PackageMeta;
+
         // Currently only used in unit tests
         #[allow(dead_code)]
         fn nevra_to_cache_branch(nevra: &CxxString) -> Result<String>;
         fn get_repodata_chksum_repr(pkg: &mut DnfPackage) -> Result<String>;
+        fn rpmts_for_commit(repo: Pin<&mut OstreeRepo>, rev: &str) -> Result<UniquePtr<RpmTs>>;
+
+        // Methods on RpmTs
+        fn packages_providing_file(self: &RpmTs, path: &str) -> Result<Vec<String>>;
+        fn package_meta(self: &RpmTs, name: &str) -> Result<UniquePtr<PackageMeta>>;
+
+        // Methods on PackageMeta
+        fn size(self: &PackageMeta) -> u64;
+        fn buildtime(self: &PackageMeta) -> u64;
     }
 
     // rpmostree-package-variants.h

rust/src/main.rs

@@ -58,6 +58,9 @@ async fn inner_async_main(args: Vec<String>) -> Result<i32> {
                 "cliwrap" => rpmostree_rust::cliwrap::entrypoint(args).map(|_| 0),
                 // The `unlock` is a hidden alias for "ostree CLI compatibility"
                 "usroverlay" | "unlock" => usroverlay(args).map(|_| 0),
+                "ex-dump-content-mapping" => {
+                    rpmostree_rust::container::content_mapping(&args_borrowed).map(|_| 0)
+                }
                 // C++ main
                 _ => Ok(rpmostree_rust::ffi::rpmostree_main(args)?),
             }

rust/src/sysroot_upgrade.rs

@@ -6,7 +6,7 @@ use crate::cxxrsutil::*;
 use crate::ffi::{output_message, ContainerImageState};
 use anyhow::Result;
 use ostree::glib;
-use ostree_container::store::LayeredImageImporter;
+use ostree_container::store::ImageImporter;
 use ostree_container::store::PrepareResult;
 use ostree_container::OstreeImageReference;
 use ostree_ext::container as ostree_container;
@@ -31,25 +31,15 @@ async fn pull_container_async(
     imgref: &OstreeImageReference,
 ) -> Result<ContainerImageState> {
     output_message(&format!("Pulling manifest: {}", &imgref));
-    let mut imp = LayeredImageImporter::new(repo, imgref, Default::default()).await?;
+    let mut imp = ImageImporter::new(repo, imgref, Default::default()).await?;
     let prep = match imp.prepare().await? {
         PrepareResult::AlreadyPresent(r) => return Ok(r.into()),
         PrepareResult::Ready(r) => r,
     };
     let digest = prep.manifest_digest.clone();
     output_message(&format!("Importing: {} (digest: {})", &imgref, &digest));
-    if prep.base_layer.commit.is_none() {
-        let size = glib::format_size(prep.base_layer.size());
-        output_message(&format!(
-            "Downloading base layer: {} ({})",
-            prep.base_layer.digest(),
-            size
-        ));
-    } else {
-        output_message(&format!("Using base: {}", prep.base_layer.digest()));
-    }
     // TODO add nice download progress
-    for layer in prep.layers.iter() {
+    for layer in prep.all_layers() {
         if layer.commit.is_some() {
             output_message(&format!("Using layer: {}", layer.digest()));
         } else {