From 06b11407d387b58733a33aa72e82fbe1fe9416c1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 26 Sep 2021 10:48:37 -0400 Subject: [PATCH] Use ostree-rs-ext (0.4.alpha) from git `main` for now For https://github.com/ostreedev/ostree-rs-ext/issues/12 Let's ship this updated code to replace the existing bits so people can try it out more easily. Once we gain experience with it, we can make a stable ostree-rs-ext release. --- Cargo.lock | 76 ++++++++++++-- Cargo.toml | 3 +- packaging/rpm-ostree.spec.in | 2 +- rust/src/lib.rs | 8 +- rust/src/sysroot_upgrade.rs | 114 +++++++++++++++------ src/daemon/rpmostree-sysroot-upgrader.cxx | 26 ++--- tests/kolainst/destructive/container-image | 19 +++- 7 files changed, 182 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 209a66576b..de7bd272b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,6 +53,19 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "async-compression" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "atty" version = "0.2.14" @@ -224,6 +237,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "containers-image-proxy" +version = "0.1.0" +source = "git+https://github.com/cgwalters/containers-image-proxy-rs#398e160f260271fdb3159d8f6db0d3f3a9d7f572" +dependencies = [ + "anyhow", + "fn-error-context", + "futures-util", + "lazy_static", + "nix 0.22.0", + "semver", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", +] + [[package]] name = "core-foundation" version = "0.9.1" @@ -1321,9 +1353,9 @@ dependencies = [ [[package]] name = "ostree" -version = "0.13.0" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9358905170a49b82e5baecb879bb944c42736a4b02074a40286a7b20665af381" +checksum = "ed97148d69c9822e948d9c1d2e288e3efd11592814981f46fce6f47cb7837d4e" dependencies = [ "bitflags", "gio", @@ -1338,20 +1370,23 @@ dependencies = [ [[package]] name = "ostree-ext" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e40c1a1b52ed5164b0fda610952438afff5162ff2dafa49e55c0a78698228d" +version = "0.4.0-alpha.0" +source = "git+https://github.com/ostreedev/ostree-rs-ext?branch=main#6a279b0de04d1cb331f60cfb75d8e1854115eca6" dependencies = [ "anyhow", + "async-compression", + "bitflags", "bytes", "camino", "cjson", + "containers-image-proxy", "flate2", "fn-error-context", "futures-util", "gvariant", "hex", "indicatif", + "lazy_static", "libc", "maplit", "nix 0.22.0", @@ -1360,6 +1395,7 @@ dependencies = [ "openssl", "ostree", "phf 0.9.0", + "pin-project", "serde", "serde_json", "structopt", @@ -1373,9 +1409,9 @@ dependencies = [ [[package]] name = "ostree-sys" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46755b75b81ac66623ee86d6eee779a82acf495a8a6a94ab458a96ad3e6880d1" +checksum = "6f21a33d095678ef9b32503ce042d4101477ab9a20c971d5f27e7f42d5a2bc79" dependencies = [ "gio-sys", "glib-sys", @@ -1509,6 +1545,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.7" @@ -1933,6 +1989,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" + [[package]] name = "serde" version = "1.0.130" diff --git a/Cargo.toml b/Cargo.toml index 88ee15286d..379ba726ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,8 @@ nix = "0.23.0" openat = "0.1.21" openat-ext = "^0.2.2" os-release = "0.1.0" -ostree-ext = "0.3.0" +# To allow us to ship the new container code as experimental +ostree-ext = { git = "https://github.com/ostreedev/ostree-rs-ext", branch = "main" } paste = "1.0" phf = { version = "0.10", features = ["macros"] } rand = "0.8.4" diff --git a/packaging/rpm-ostree.spec.in b/packaging/rpm-ostree.spec.in index f77ee56c72..16a93c3c93 100644 --- a/packaging/rpm-ostree.spec.in +++ b/packaging/rpm-ostree.spec.in @@ -48,7 +48,7 @@ BuildRequires: gnome-common BuildRequires: /usr/bin/g-ir-scanner # Core requirements # One way to check this: `objdump -p /path/to/rpm-ostree | grep LIBOSTREE` and pick the highest (though that might miss e.g. new struct members) -BuildRequires: pkgconfig(ostree-1) >= 2021.1 +BuildRequires: pkgconfig(ostree-1) >= 2021.5 BuildRequires: pkgconfig(polkit-gobject-1) BuildRequires: pkgconfig(json-glib-1.0) BuildRequires: pkgconfig(rpm) >= 4.14.0 diff --git a/rust/src/lib.rs b/rust/src/lib.rs index bbf041ae49..8c2efc325d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -137,19 +137,19 @@ pub mod ffi { /// cxx.rs currently requires types used as extern Rust types to be defined by the same crate /// that contains the bridge using them, so we redefine an `ContainerImport` struct here. pub(crate) struct ContainerImport { + pub changed: bool, pub ostree_commit: String, pub image_digest: String, } // sysroot_upgrade.rs extern "Rust" { - fn import_container( + fn pull_container( repo: Pin<&mut OstreeRepo>, cancellable: Pin<&mut GCancellable>, - imgref: String, + imgref: &str, + current_digest: &str, ) -> Result>; - - fn fetch_digest(imgref: String, cancellable: Pin<&mut GCancellable>) -> Result; } // core.rs diff --git a/rust/src/sysroot_upgrade.rs b/rust/src/sysroot_upgrade.rs index a46468d5c4..ad56cb9a2e 100644 --- a/rust/src/sysroot_upgrade.rs +++ b/rust/src/sysroot_upgrade.rs @@ -3,48 +3,102 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::cxxrsutil::*; -use crate::ffi::ContainerImport; -use std::convert::TryInto; +use crate::ffi::{output_message, ContainerImport}; +use anyhow::Result; +use ostree::glib; +use ostree_container::store::LayeredImageImporter; +use ostree_container::store::PrepareResult; +use ostree_container::OstreeImageReference; +use ostree_ext::container as ostree_container; +use ostree_ext::ostree; +use std::convert::TryFrom; use std::pin::Pin; use tokio::runtime::Handle; +async fn pull_container_async( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + current_digest: Option<&str>, +) -> Result { + let unchanged = || ContainerImport { + changed: false, + ostree_commit: "".to_string(), + image_digest: "".to_string(), + }; + output_message(&format!("Pulling manifest: {}", &imgref)); + match &imgref.sigverify { + ostree_container::SignatureSource::OstreeRemote(_) => { + let (_, digest) = ostree_container::fetch_manifest(&imgref).await?; + if Some(digest.as_str()) == current_digest { + return Ok(unchanged()); + } + output_message(&format!("Importing: {} (digest: {})", &imgref, &digest)); + let i = ostree_container::import(repo, imgref, None).await?; + Ok(ContainerImport { + changed: true, + ostree_commit: i.ostree_commit, + image_digest: i.image_digest, + }) + } + _ => { + let mut imp = LayeredImageImporter::new(repo, imgref).await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(_) => return Ok(unchanged()), + PrepareResult::Ready(r) => r, + }; + let digest = prep.manifest_digest.clone(); + output_message(&format!("Importing: {} (digest: {})", &imgref, &digest)); + if prep.base_layer.commit.is_none() { + let size = glib::format_size(prep.base_layer.size()); + output_message(&format!( + "Downloading base layer: {} ({})", + prep.base_layer.digest(), + size + )); + } else { + output_message(&format!("Using base: {}", prep.base_layer.digest())); + } + // TODO add nice download progress + for layer in prep.layers.iter() { + if layer.commit.is_some() { + output_message(&format!("Using layer: {}", layer.digest())); + } else { + let size = glib::format_size(layer.size()); + output_message(&format!("Downloading layer: {} ({})", layer.digest(), size)); + } + } + let import = imp.import(prep).await?; + Ok(ContainerImport { + changed: true, + ostree_commit: import.commit, + image_digest: digest, + }) + } + } +} + /// Import ostree commit in container image using ostree-rs-ext's API. -pub(crate) fn import_container( +pub(crate) fn pull_container( mut repo: Pin<&mut crate::FFIOstreeRepo>, mut cancellable: Pin<&mut crate::FFIGCancellable>, - imgref: String, + imgref: &str, + current_digest: &str, ) -> CxxResult> { - let repo = repo.gobj_wrap(); - let cancellable = cancellable.gobj_wrap(); - let imgref = imgref.as_str().try_into()?; - - let imported = Handle::current().block_on(async { - crate::utils::run_with_cancellable( - async { ostree_ext::container::import(&repo, &imgref, None).await }, - &cancellable, - ) - .await - })?; - Ok(Box::new(ContainerImport { - ostree_commit: imported.ostree_commit, - image_digest: imported.image_digest, - })) -} - -/// Fetch the image digest for `imgref` using ostree-rs-ext's API. -pub(crate) fn fetch_digest( - imgref: String, - mut cancellable: Pin<&mut crate::FFIGCancellable>, -) -> CxxResult { - let imgref = imgref.as_str().try_into()?; + let repo = &repo.gobj_wrap(); let cancellable = cancellable.gobj_wrap(); + let current_digest = if current_digest.is_empty() { + None + } else { + Some(current_digest) + }; + let imgref = &OstreeImageReference::try_from(imgref)?; - let digest = Handle::current().block_on(async { + let r = Handle::current().block_on(async { crate::utils::run_with_cancellable( - async { ostree_ext::container::fetch_manifest_info(&imgref).await }, + async { pull_container_async(repo, imgref, current_digest).await }, &cancellable, ) .await })?; - Ok(digest.manifest_digest) + Ok(Box::new(r)) } diff --git a/src/daemon/rpmostree-sysroot-upgrader.cxx b/src/daemon/rpmostree-sysroot-upgrader.cxx index ac435f9297..434b0134be 100644 --- a/src/daemon/rpmostree-sysroot-upgrader.cxx +++ b/src/daemon/rpmostree-sysroot-upgrader.cxx @@ -450,26 +450,16 @@ rpmostree_sysroot_upgrader_pull_base (RpmOstreeSysrootUpgrader *self, if (override_commit) return glnx_throw (error, "Specifying commit overrides for container-image-reference type refspecs is not supported"); - if (strstr (refspec, "@")) - return glnx_throw (error, "Pinned to specific container image digest, cannot upgrade"); - - /* Check to see if the digest that `refspec` points to is the same before pulling a new container image. */ - const char *cur_digest = rpmostree_origin_get_container_image_reference_digest (self->original_origin); - if (cur_digest) + auto refspec_s = std::string(refspec); + const char *cur_digest_c = rpmostree_origin_get_container_image_reference_digest (self->original_origin); + auto cur_digest = std::string(cur_digest_c ?: ""); + auto import = rpmostreecxx::pull_container(*self->repo, *cancellable, refspec_s, cur_digest); + if (!import->changed) { - rpmostree_output_message ("Pulling manifest: %s", refspec); - auto new_digest = CXX_TRY_VAL(fetch_digest(std::string(refspec), *cancellable), error); - if (strcmp (new_digest.c_str(), cur_digest) == 0) - { - /* No new digest. */ - *out_changed = FALSE; - return TRUE; - } + /* No new digest. */ + *out_changed = FALSE; + return TRUE; } - - rpmostree_output_message ("Pulling: %s", refspec); - auto import = CXX_TRY_VAL(import_container(*self->repo, *cancellable, std::string(refspec)), error); - rpmostree_origin_set_container_image_reference_digest (self->original_origin, import->image_digest.c_str()); new_base_rev = strdup (import->ostree_commit.c_str()); break; diff --git a/tests/kolainst/destructive/container-image b/tests/kolainst/destructive/container-image index d679fe6f21..0a7ac5f879 100755 --- a/tests/kolainst/destructive/container-image +++ b/tests/kolainst/destructive/container-image @@ -26,18 +26,23 @@ set -x libtest_prepare_offline cd $(mktemp -d) -image=containers-storage:localhost/fcos:latest +basedir=/var/tmp/fcos +# TODO: It'd be much better to test this via a registry +image=oci:$basedir image_pull=ostree-unverified-image:$image case "${AUTOPKGTEST_REBOOT_MARK:-}" in "") # Test rebase checksum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum') + content_checksum=$(ostree show "${checksum}" | grep ContentChecksum) rpm-ostree ex-container export --repo=/ostree/repo ${checksum} "${image}" rpm-ostree rebase "$image_pull" --experimental | tee out.txt - assert_file_has_content_literal out.txt 'Pulling: '"$image_pull" - rpmostree_assert_status ".deployments[0][\"container-image-reference\"] == \"ostree-unverified-image:containers-storage:localhost/fcos:latest\"" - rpmostree_assert_status ".deployments[0][\"checksum\"] == \"${checksum}\"" + assert_file_has_content out.txt 'Importing.*'"$image_pull" + rpmostree_assert_status ".deployments[0][\"container-image-reference\"] == \"ostree-unverified-image:$image\"" + new_checksum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum') + new_content_checksum=$(ostree show "${new_checksum}" | grep ContentChecksum) + assert_streq "${content_checksum}" "${new_content_checksum}" echo "ok rebase to container image reference" rpm-ostree status | tee out.txt @@ -69,11 +74,15 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in with_foo_commit=$(rpm-ostree status --json | jq -r '.deployments[0].checksum') ostree refs ${with_foo_commit} --create vmcheck_tmp/new_update new_commit=$(ostree commit -b vmcheck --tree=ref=vmcheck_tmp/new_update) + new_content_checksum=$(ostree show "${new_commit}" | grep ContentChecksum) + rm "$basedir" -rf rpm-ostree ex-container export --repo=/ostree/repo ${new_commit} "$image" rpm-ostree uninstall foo rpm-ostree upgrade - rpmostree_assert_status ".deployments[0][\"checksum\"] == \"${new_commit}\"" + new_client_checksum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum') + new_client_content_checksum=$(ostree show "${new_client_checksum}" | grep ContentChecksum) + assert_streq "${new_content_checksum}" "${new_client_content_checksum}" /tmp/autopkgtest-reboot 2 ;;