Move install tests shell script into Rust

A few things going on here:

- Rewrite logic from shell script into Rust (using xshell, so
  it's still convenient to fork commands)
- Make the test logic take an externally-built container image
  instead of using a `-v bootc:/usr/bin/bootc` bind mount
- Build the container image using our stock hack/Containerfile
  in Github Actions instead of building for c9s in GHA
- This all hence starts to make the logic reusable outside
  of Github Actions too; the container build is a known standard thing.

Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/workflows/ci.yml

@@ -68,26 +68,6 @@ jobs:
         with:
           name: bootc.tar.zst
           path: target/bootc.tar.zst
-  build-c9s:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
-    runs-on: ubuntu-latest
-    container: quay.io/centos/centos:stream9
-    steps:
-      - run: dnf -y install git-core
-      - uses: actions/checkout@v4
-      - name: Install deps
-        run: ./ci/installdeps.sh
-      - name: Cache Dependencies
-        uses: Swatinem/rust-cache@v2
-        with:
-          key: "build-c9s"
-      - name: Build
-        run: make test-bin-archive
-      - name: Upload binary
-        uses: actions/upload-artifact@v4
-        with:
-          name: bootc-c9s.tar.zst
-          path: target/bootc.tar.zst
   cargo-deny:
     runs-on: ubuntu-latest
     steps:
@@ -127,78 +107,21 @@ jobs:
         run: sudo tar -C / -xvf bootc.tar.zst
       - name: Integration tests
         run: bootc internal-tests run-container-integration
-  privtest-alongside:
+  install-tests:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
-    name: "Test install-alongside"
-    needs: [build-c9s]
-    runs-on: ubuntu-latest
+    name: "Test install"
+    # For a not-ancient podman
+    runs-on: ubuntu-24.04
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
       - name: Ensure host skopeo is disabled
         run: sudo rm -f /bin/skopeo /usr/bin/skopeo
-      - name: Download
-        uses: actions/download-artifact@v4
-        with:
-          name: bootc-c9s.tar.zst
-      - name: Install
-        run: tar -xvf bootc.tar.zst
-      - name: Integration tests
-        run: |
-          set -xeuo pipefail
-          image=quay.io/centos-bootc/centos-bootc-dev:stream9 
-          echo 'ssh-ed25519 ABC0123 testcase@example.com' > test_authorized_keys
-          sudo podman run --rm --privileged -v ./test_authorized_keys:/test_authorized_keys --env RUST_LOG=debug -v /:/target -v /var/lib/containers:/var/lib/containers -v ./usr/bin/bootc:/usr/bin/bootc --pid=host --security-opt label=disable \
-            ${image} bootc install to-filesystem --acknowledge-destructive \
-            --karg=foo=bar --disable-selinux --replace=alongside --root-ssh-authorized-keys=/test_authorized_keys /target
-          ls -al /boot/loader/
-          sudo grep foo=bar /boot/loader/entries/*.conf
-          grep authorized_keys /ostree/deploy/default/deploy/*/etc/tmpfiles.d/bootc-root-ssh.conf
-          # TODO fix https://github.com/containers/bootc/pull/137
-          sudo chattr -i /ostree/deploy/default/deploy/*
-          sudo rm /ostree/deploy/default -rf
-          sudo podman run --rm --privileged --env RUST_LOG=debug -v /:/target -v /var/lib/containers:/var/lib/containers -v ./usr/bin/bootc:/usr/bin/bootc --pid=host --security-opt label=disable \
-            ${image} bootc install to-existing-root --acknowledge-destructive
-          sudo podman run --rm --privileged -v /:/target -v ./usr/bin/bootc:/usr/bin/bootc --pid=host --security-opt label=disable ${image} bootc internal-tests verify-selinux /target/ostree --warn
-  install-to-existing-root:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
-    name: "Test install-to-existing-root"
-    needs: [build-c9s]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download
-        uses: actions/download-artifact@v4
-        with:
-          name: bootc-c9s.tar.zst
-      - name: Install
-        run: tar -xvf bootc.tar.zst
-      - name: Integration tests
-        run: |
-          set -xeuo pipefail
-          # We should be able to install to-existing-root with no install config,
-          # so we bind mount an empty directory over /usr/lib/bootc/install.
-          empty=$(mktemp -d)
-          image=quay.io/centos-bootc/centos-bootc-dev:stream9
-          sudo podman run --rm --privileged --env RUST_LOG=debug -v /:/target -v /var/lib/containers:/var/lib/containers -v ./usr/bin/bootc:/usr/bin/bootc -v ${empty}:/usr/lib/bootc/install --pid=host --security-opt label=disable \
-            ${image} bootc install to-existing-root
-  install-to-loopback:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
-    name: "Test install to-disk --via-loopback"
-    needs: [build-c9s]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download
-        uses: actions/download-artifact@v4
-        with:
-          name: bootc-c9s.tar.zst
-      - name: Install
-        run: tar -xvf bootc.tar.zst
       - name: Integration tests
         run: |
-          set -xeuo pipefail
-          image=quay.io/centos-bootc/centos-bootc-dev:stream9
-          tmpdisk=$(mktemp -p /var/tmp)
-          truncate -s 20G ${tmpdisk}
-          sudo podman run --rm --privileged --env RUST_LOG=debug -v /dev:/dev -v /:/target -v /var/lib/containers:/var/lib/containers -v ./usr/bin/bootc:/usr/bin/bootc --pid=host --security-opt label=disable \
-            -v ${tmpdisk}:/disk ${image} bootc install to-disk --via-loopback /disk
+          set -xeu
+          sudo podman build -t localhost/bootc -f hack/Containerfile .
+          cargo run -p tests-integration run-install-tests localhost/bootc
   docs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }}
     runs-on: ubuntu-latest

Cargo.lock

@@ -1,5 +1,5 @@
 [workspace]
-members = ["cli", "lib", "xtask"]
+members = ["cli", "lib", "xtask", "tests-integration"]
 resolver = "2"
 
 [profile.dev]

Cargo.toml

@@ -7,7 +7,7 @@ WORKDIR /build
 RUN mkdir -p /build/target/dev-rootfs  # This can hold arbitrary extra content
 # See https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/
 # We aren't using the full recommendations there, just the simple bits.
-RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome make bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
+RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome make test-bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
 
 FROM quay.io/centos-bootc/centos-bootc:stream9
 COPY --from=build /out/bootc.tar.zst /tmp

hack/Containerfile

@@ -15,8 +15,7 @@ pub fn generate_manpages(directory: &Utf8Path) -> Result<()> {
 fn generate_one(directory: &Utf8Path, cmd: Command) -> Result<()> {
     let version = env!("CARGO_PKG_VERSION");
     let name = cmd.get_name();
-    let bin_name = cmd.get_bin_name()
-        .unwrap_or_else(|| name);
+    let bin_name = cmd.get_bin_name().unwrap_or_else(|| name);
     let path = directory.join(format!("{name}.8"));
     println!("Generating {path}...");
 
@@ -37,12 +36,13 @@ fn generate_one(directory: &Utf8Path, cmd: Command) -> Result<()> {
 
     for subcmd in cmd.get_subcommands().filter(|c| !c.is_hide_set()) {
         let subname = format!("{}-{}", name, subcmd.get_name());
-        let bin_name =  format!("{} {}", bin_name, subcmd.get_name());
+        let bin_name = format!("{} {}", bin_name, subcmd.get_name());
         // SAFETY: Latest clap 4 requires names are &'static - this is
         // not long-running production code, so we just leak the names here.
         let subname = &*std::boxed::Box::leak(subname.into_boxed_str());
         let bin_name = &*std::boxed::Box::leak(bin_name.into_boxed_str());
-        let subcmd = subcmd.clone()
+        let subcmd = subcmd
+            .clone()
             .name(subname)
             .alias(subname)
             .bin_name(bin_name)

lib/src/docgen.rs

@@ -0,0 +1,20 @@
+# Our integration tests
+[package]
+name = "tests-integration"
+version = "0.1.0"
+license = "MIT OR Apache-2.0"
+edition = "2021"
+publish = false
+
+[[bin]]
+name = "tests-integration"
+path = "src/tests-integration.rs"
+
+[dependencies]
+anyhow = "1.0.82"
+camino = "1.1.6"
+cap-std-ext = "4"
+clap = { version= "4.5.4", features = ["derive","cargo"] }
+fn-error-context = "0.2.1"
+tempfile = "3.10.1"
+xshell = { version = "0.2.6" }

tests-integration/Cargo.toml

@@ -0,0 +1,157 @@
+use std::os::fd::AsRawFd;
+use std::path::Path;
+
+use anyhow::Result;
+use cap_std_ext::cap_std;
+use cap_std_ext::cap_std::fs::Dir;
+use clap::Parser;
+use fn_error_context::context;
+
+use xshell::{cmd, Shell};
+
+#[derive(Debug, Parser, PartialEq, Eq)]
+#[clap(name = "bootc-integration-tests", version, rename_all = "kebab-case")]
+pub(crate) enum Opt {
+    RunInstallTests {
+        /// Source container image reference
+        image: String,
+    },
+}
+
+fn main() {
+    if let Err(e) = try_main() {
+        eprintln!("error: {e:?}");
+        std::process::exit(1);
+    }
+}
+
+fn try_main() -> Result<()> {
+    let opt = Opt::parse();
+    match opt {
+        Opt::RunInstallTests { image } => run_install_tests(image.as_str()),
+    }
+}
+
+// Clear out and delete any ostree roots
+fn reset_root(sh: &Shell) -> Result<()> {
+    // TODO fix https://github.com/containers/bootc/pull/137
+    if !Path::new("/ostree/deploy/default").exists() {
+        return Ok(());
+    }
+    cmd!(
+        sh,
+        "sudo /bin/sh -c 'chattr -i /ostree/deploy/default/deploy/*'"
+    )
+    .run()?;
+    cmd!(sh, "sudo rm /ostree/deploy/default -rf").run()?;
+    Ok(())
+}
+
+fn find_deployment_root() -> Result<Dir> {
+    let _stateroot = "default";
+    let d = Dir::open_ambient_dir(
+        "/ostree/deploy/default/deploy",
+        cap_std::ambient_authority(),
+    )?;
+    for child in d.entries()? {
+        let child = child?;
+        if !child.file_type()?.is_dir() {
+            continue;
+        }
+        return Ok(child.open_dir()?);
+    }
+    anyhow::bail!("Failed to find deployment root")
+}
+
+fn run_test<F>(sh: &Shell, desc: &str, f: F) -> Result<()>
+where
+    F: FnOnce(&Shell) -> Result<()>,
+{
+    reset_root(sh)?;
+    println!("test: {desc}");
+    match f(sh) {
+        Ok(r) => {
+            println!("ok: {desc}");
+            Ok(r)
+        }
+        Err(e) => {
+            eprintln!("FAILED: {desc}");
+            Err(e)
+        }
+    }
+}
+
+#[context("Install tests")]
+fn run_install_tests(image: &str) -> Result<()> {
+    let sh = &xshell::Shell::new()?;
+
+    let base_args = [
+        "podman",
+        "run",
+        "--rm",
+        "--privileged",
+        "-v",
+        "/dev:/dev",
+        "-v",
+        "/var/lib/containers:/var/lib/containers",
+        "--pid=host",
+        "--security-opt",
+        "label=disable",
+    ];
+    let image_install = [image, "bootc", "install"];
+    let target_args = ["-v", "/:/target"];
+    // We always need this as we assume we're operating on a local image
+    let generic_inst_args = ["--skip-fetch-check"];
+
+    run_test(sh, "loopback install", |sh| {
+        let size = 10 * 1000 * 1000 * 1000;
+        let mut tmpdisk = tempfile::NamedTempFile::new_in("/var/tmp")?;
+        tmpdisk.as_file_mut().set_len(size)?;
+        let tmpdisk = tmpdisk.into_temp_path();
+        let tmpdisk = tmpdisk.to_str().unwrap();
+        cmd!(sh, "sudo {base_args...} -v {tmpdisk}:/disk {image_install...} to-disk --via-loopback {generic_inst_args...} /disk").run()?;
+        Ok(())
+    })?;
+
+    run_test(
+        sh,
+        "replace=alongside with ssh keys and a karg, and SELinux disabled",
+        |sh| {
+            let tmpd = &sh.create_temp_dir()?;
+            let tmp_keys = tmpd.path().join("test_authorized_keys");
+            let tmp_keys = tmp_keys.to_str().unwrap();
+            std::fs::write(&tmp_keys, b"ssh-ed25519 ABC0123 testcase@example.com")?;
+            cmd!(sh, "sudo {base_args...} {target_args...} -v {tmp_keys}:/test_authorized_keys {image_install...} to-filesystem {generic_inst_args...} --acknowledge-destructive --karg=foo=bar --replace=alongside --root-ssh-authorized-keys=/test_authorized_keys /target").run()?;
+
+            cmd!(
+                sh,
+                "sudo /bin/sh -c 'grep foo=bar /boot/loader/entries/*.conf'"
+            )
+            .run()?;
+            let deployment = &find_deployment_root()?;
+            let cwd = sh.push_dir(format!("/proc/self/fd/{}", deployment.as_raw_fd()));
+            cmd!(
+                sh,
+                "grep authorized_keys etc/tmpfiles.d/bootc-root-ssh.conf"
+            )
+            .run()?;
+            drop(cwd);
+            Ok(())
+        },
+    )?;
+
+    run_test(sh, "Install and verify selinux state", |sh| {
+        cmd!(sh, "sudo {base_args...} {target_args...} {image_install...} to-existing-root --acknowledge-destructive {generic_inst_args...}").run()?;
+        cmd!(sh, "sudo podman run --rm --privileged --pid=host {target_args...} {image} bootc internal-tests verify-selinux /target/ostree --warn").run()?;
+        Ok(())
+    })?;
+
+    run_test(sh, "without an install config", |sh| {
+        let empty = sh.create_temp_dir()?;
+        let empty = empty.path().to_str().unwrap();
+        cmd!(sh, "sudo {base_args...} {target_args...} -v {empty}:/usr/lib/bootc/install {image_install...} to-existing-root {generic_inst_args...}").run()?;
+        Ok(())
+    })?;
+
+    Ok(())
+}