feat: libvirt/KVM infrastructure backend as an alternative to DigitalOcean

.env.example

@@ -1,11 +1,27 @@
+# --- Infrastructure backend ---
+# digitalocean (default): flintlock hosts are DO droplets; needs DO_API_TOKEN.
+# libvirt: flintlock hosts are local KVM VMs; see docs/libvirt-backend.md.
+INFRA_BACKEND=digitalocean
+
 # --- DigitalOcean ---
-# required: DigitalOcean API token
+# required when INFRA_BACKEND=digitalocean: DigitalOcean API token
 DO_API_TOKEN=
 DO_REGION=nyc3            # must offer nested-virt Basic droplets + block storage
 DO_DROPLET_SIZE=s-4vcpu-8gb
 DO_IMAGE=ubuntu-22-04-x64
 DO_BLOCK_VOLUME_GB=50     # raw volume per droplet for the flintlock thinpool
 
+# --- libvirt (only when INFRA_BACKEND=libvirt) ---
+LIBVIRT_URI=qemu:///system
+LIBVIRT_POOL=lm-acceptance          # storage pool; created on first run
+LIBVIRT_SUBNET_PREFIX=10.210        # each run takes the first free <prefix>.N.0/24
+LIBVIRT_VCPUS=4
+LIBVIRT_MEMORY_MB=8192
+LIBVIRT_DISK_GB=50
+# Pinned Ubuntu 22.04 cloud image; leave both unset to use the built-in pin.
+# LIBVIRT_BASE_IMAGE_URL=
+# LIBVIRT_BASE_IMAGE_SHA256=
+
 # --- Run identity ---
 # optional; auto-generated as at-<8hex> if empty
 RUN_ID=

.github/workflows/e2e-libvirt.yml

@@ -0,0 +1,106 @@
+name: e2e-libvirt
+
+# Manually triggered end-to-end run against local KVM VMs on a self-hosted bare-metal
+# runner (INFRA_BACKEND=libvirt). No DigitalOcean resources or secrets are involved.
+# See docs/libvirt-backend.md for the runner prerequisites.
+on:
+  workflow_dispatch:
+    inputs:
+      suite:
+        description: Which suite to run
+        type: choice
+        options: [brigade, battery, both]
+        default: brigade
+      brigade_ref:
+        description: brigade git ref to test
+        default: main
+      flintlock_ref:
+        description: flintlock git ref to test
+        default: main
+      battery_ref:
+        description: battery release tag to test
+        default: v0.3.2
+      node_count:
+        description: Number of flintlock host VMs
+        default: "2"
+      microvm_kernel_image:
+        description: microVM kernel OCI image
+        default: ghcr.io/liquidmetal-dev/firecracker-kernel:5.10-no-acpi
+      microvm_rootfs_image:
+        description: microVM rootfs OCI image
+        default: ghcr.io/liquidmetal-dev/ubuntu:24.04
+      microvm_provider:
+        description: Hypervisor provider (firecracker | cloudhypervisor)
+        default: firecracker
+
+# One run at a time on the box: it holds one 2-node run comfortably, and serial runs are
+# what make the "sweep everything" steps below safe. Separate from the DigitalOcean group
+# so the two never queue behind each other.
+concurrency:
+  group: e2e-libvirt
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: [self-hosted, linux, x64, kvm]
+    timeout-minutes: 150
+    env:
+      INFRA_BACKEND: libvirt
+      BRIGADE_REF: ${{ inputs.brigade_ref }}
+      FLINTLOCK_REF: ${{ inputs.flintlock_ref }}
+      BATTERY_REF: ${{ inputs.battery_ref }}
+      NODE_COUNT: ${{ inputs.node_count }}
+      MICROVM_KERNEL_IMAGE: ${{ inputs.microvm_kernel_image }}
+      MICROVM_ROOTFS_IMAGE: ${{ inputs.microvm_rootfs_image }}
+      MICROVM_PROVIDER: ${{ inputs.microvm_provider }}
+    steps:
+      - name: Preflight — KVM and libvirt
+        run: |
+          test -e /dev/kvm || { echo "::error::/dev/kvm not present on this runner" >&2; exit 1; }
+          virsh -c qemu:///system version || {
+            echo "::error::cannot reach system libvirt; see docs/libvirt-backend.md" >&2; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: make venv
+
+      # A cancelled or crashed earlier job can leave VMs behind on this persistent runner.
+      - name: Sweep stale libvirt resources
+        run: make clean-libvirt
+
+      - name: Generate ephemeral SSH keypair
+        run: |
+          # In RUNNER_TEMP, not ~/.ssh: the runner is persistent and this key is per-run.
+          ssh-keygen -t ed25519 -N "" -C "lm-acceptance-${GITHUB_RUN_ID}" -f "${RUNNER_TEMP}/id_ed25519"
+          echo "SSH_PRIVATE_KEY_PATH=${RUNNER_TEMP}/id_ed25519" >> "${GITHUB_ENV}"
+          echo "SSH_PUBLIC_KEY_PATH=${RUNNER_TEMP}/id_ed25519.pub" >> "${GITHUB_ENV}"
+
+      - name: Run brigade suite
+        if: ${{ inputs.suite == 'brigade' || inputs.suite == 'both' }}
+        run: make test
+
+      - name: Run battery suite
+        if: ${{ !cancelled() && (inputs.suite == 'battery' || inputs.suite == 'both') }}
+        run: make test-battery
+
+      - name: Clean up libvirt resources
+        if: always()
+        run: make clean-libvirt
+
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: artifacts-libvirt-${{ github.run_id }}
+          path: artifacts/**
+          if-no-files-found: ignore

AGENTS.md

@@ -5,7 +5,8 @@ Guide for coding agents working in this repo. See `README.md` for the full human
 ## Overview
 
 End-to-end acceptance suite for **flintlock** microVM orchestration, running on real
-**DigitalOcean** infrastructure. Two independent suites:
+**DigitalOcean** infrastructure, or on local KVM VMs via libvirt (`INFRA_BACKEND=libvirt`,
+see `docs/libvirt-backend.md`). Two independent suites:
 
 - **brigade** (`tests/`) — a run provisions a VPC + SSH key + 2 droplets (2 flintlock hosts) +
   block volumes + firewall, bootstraps containerd/Firecracker/`flintlockd` and a 2-node
@@ -40,6 +41,8 @@ make proto                               # regenerate gRPC stubs (flintlock + ba
 make refresh-proto                       # re-fetch + revendor upstream flintlock protos
 make refresh-battery-proto               # re-fetch + revendor battery's own protos
 make clean-tags                          # reap leftover at-* / lm-acceptance-* DO resources
+INFRA_BACKEND=libvirt make test          # brigade e2e on local KVM VMs (no DO, no cost)
+make clean-libvirt                       # reap leftover lm-acceptance-* libvirt VMs/networks
 ```
 
 Config is entirely env-driven — see `.env.example` for every knob. `RUN_ID` (auto `at-<hex>`)
@@ -51,7 +54,8 @@ host `journalctl` is always collected to `artifacts/<run_id>/`.
 ```
 liquidmetal_at/
   config.py            env → Config, RUN_ID, validation
-  infra/               DigitalOcean provisioning (do.py) + teardown (reaper.py)
+  infra/               backends: DigitalOcean (do.py + reaper.py), libvirt (libvirt.py);
+                       backend.py selects
   bootstrap/           host + brigade + battery bootstrap, Jinja2 templates/
   flintlock/           gRPC client + generated stubs (gen/, shared with battery/)
   battery/             gRPC client to poolmgrd (PoolAdmin/Lease/Events) + PoolSpec builder

Makefile

@@ -54,6 +54,12 @@ lint:
 clean-tags:
 	$(PY) -m liquidmetal_at.infra.reaper
 
+# Delete any libvirt VMs, networks and overlay volumes left by local/self-hosted runs
+# (lm-acceptance-*). Base images are kept.
+.PHONY: clean-libvirt
+clean-libvirt:
+	$(PY) -m liquidmetal_at.infra.libvirt
+
 .PHONY: clean
 clean:
 	rm -rf $(GEN)/flapi $(GEN)/fltypes $(GEN)/poolmgr $(GEN)/__init__.py

README.md

@@ -18,6 +18,9 @@ Each run:
 > **Nested virtualization:** flintlock needs `/dev/kvm`. DigitalOcean Basic droplets expose
 > nested virt (as of 2026) but performance is poor — timeouts are sized generously.
 
+> **No DigitalOcean account?** Set `INFRA_BACKEND=libvirt` to run the same suites against
+> local KVM VMs on a bare-metal Linux machine. See [docs/libvirt-backend.md](docs/libvirt-backend.md).
+
 ## Layout
 
 ```
@@ -131,6 +134,7 @@ Teardown is automatic. If a run is killed, reap leftovers:
 
 ```bash
 make clean-tags           # deletes all lm-acceptance-* tagged DO resources
+make clean-libvirt        # deletes all lm-acceptance-* libvirt VMs, networks and overlays
 ```
 
 Set `KEEP_INFRA_ON_FAILURE=true` to leave infra up for debugging when a test fails; host

docs/libvirt-backend.md

@@ -0,0 +1,101 @@
+# libvirt backend (local KVM)
+
+`INFRA_BACKEND=libvirt` runs both suites against local KVM virtual machines instead of
+DigitalOcean droplets. It is meant for a bare-metal self-hosted GitHub runner and for a
+developer workstation. No DigitalOcean token is needed.
+
+Each run creates, under system libvirt (`qemu:///system`):
+
+- a NAT network `lm-acceptance-<run_id>` on the first free `10.210.N.0/24`;
+- `NODE_COUNT` VMs `lm-acceptance-<run_id>-host<i>` (4 vCPU / 8 GB / 50 GB by default), each
+  a qcow2 overlay on a shared Ubuntu 22.04 cloud image;
+- one serial console log per VM.
+
+The base image is downloaded once to `~/.cache/lm-acceptance/`, verified against a pinned
+SHA256, and stored in the `lm-acceptance` storage pool
+(`/var/lib/libvirt/images/lm-acceptance`). It is reused across runs and never deleted
+automatically.
+
+## Requirements
+
+- Bare metal x86-64 with hardware virtualization. The flintlock hosts run Firecracker /
+  Cloud Hypervisor microVMs *inside* the VMs, so nested virtualization must be on.
+- For the default two nodes: about 8 cores, 16 GB RAM and 100 GB of free disk.
+- Internet access from the VMs (packages, toolchains, OCI images).
+
+## Setup
+
+### Ubuntu LTS (runner)
+
+```bash
+sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients virtinst
+sudo usermod -aG libvirt,kvm "$USER"      # the user the runner service runs as; re-login
+```
+
+### Arch (workstation)
+
+```bash
+sudo pacman -S --needed qemu-base libvirt virt-install dnsmasq
+sudo systemctl enable --now libvirtd.socket
+sudo usermod -aG libvirt "$USER"          # then log out and back in
+```
+
+### Nested virtualization
+
+```bash
+cat /sys/module/kvm_intel/parameters/nested   # Intel: Y or 1
+cat /sys/module/kvm_amd/parameters/nested     # AMD: 1
+```
+
+If it is off, enable it and reload the module (or reboot):
+
+```bash
+echo "options kvm_intel nested=1" | sudo tee /etc/modprobe.d/kvm-nested.conf   # or kvm_amd
+```
+
+### Host firewall
+
+With **ufw** (or another default-deny firewall) active and libvirt on its nftables
+firewall backend, the VMs get no DHCP lease and no DNS: libvirt's accept rules live in a
+separate table, so the firewall still drops the packets. Provisioning then fails with
+"got no DHCP lease". Switch libvirt to the iptables backend:
+
+```bash
+sudo sed -i 's/^#\?firewall_backend *=.*/firewall_backend = "iptables"/' /etc/libvirt/network.conf
+sudo systemctl restart libvirtd
+```
+
+**Docker** sets the iptables `FORWARD` policy to `DROP`, which can break the VMs' outbound
+access. Keep Docker off the runner.
+
+## Running
+
+```bash
+INFRA_BACKEND=libvirt make test           # brigade suite
+INFRA_BACKEND=libvirt make test-battery   # battery suite
+```
+
+Or set `INFRA_BACKEND=libvirt` in `.env`. The other knobs (`LIBVIRT_*`) are listed in
+`.env.example`; `NODE_COUNT` applies to both backends.
+
+In CI, dispatch the **e2e-libvirt** workflow and pick a suite. The runner must carry the
+labels `self-hosted`, `linux`, `x64`, `kvm`.
+
+## Debugging and cleanup
+
+- Host journals and diagnostics are collected to `artifacts/<run_id>/` as with
+  DigitalOcean, plus `host<i>-console.log`, each VM's serial console. The console is the
+  only evidence when a VM never boots or never gets an address.
+- `KEEP_INFRA_ON_FAILURE=true` leaves the VMs up after a failed run:
+  `ssh -i <key> root@10.210.N.10`, `virsh -c qemu:///system list --all`.
+- `make clean-libvirt` removes every `lm-acceptance-*` VM, network and overlay volume,
+  including kept ones. Base images stay; remove one with
+  `virsh -c qemu:///system vol-delete --pool lm-acceptance <name>`.
+
+## Bumping the base image
+
+Pick a dated release under <https://cloud-images.ubuntu.com/releases/jammy/>, take the
+`ubuntu-22.04-server-cloudimg-amd64.img` line from its `SHA256SUMS`, and update the two
+defaults in `liquidmetal_at/config.py` (or set `LIBVIRT_BASE_IMAGE_URL` and
+`LIBVIRT_BASE_IMAGE_SHA256`). The new image becomes a new base volume; the old one stays
+until deleted by hand.

liquidmetal_at/bootstrap/brigade_cluster.py

@@ -10,15 +10,15 @@
 
 from .. import brigade_status
 from ..config import Config
-from ..infra.do import Infra
+from ..infra.types import Infra
 from ..waiter import wait_until
 
 log = logging.getLogger("cluster")
 
 
 def wait_for_cluster(cfg: Config, infra: Infra) -> None:
     target = cfg.brigade_min_cluster_size
-    for d in infra.droplets:
+    for d in infra.nodes:
         def _formed(ip=d.public_ip) -> bool:
             size = brigade_status.cluster_size(ip, cfg.brigade_status_port)
             log.info("node %s reports cluster size %d (want >= %d)", ip, size, target)
@@ -30,4 +30,4 @@ def _formed(ip=d.public_ip) -> bool:
             interval=5,
             description=f"brigade cluster size>={target} on {d.public_ip}",
         )
-    log.info("brigade cluster formed (size>=%d) across %d nodes", target, len(infra.droplets))
+    log.info("brigade cluster formed (size>=%d) across %d nodes", target, len(infra.nodes))

liquidmetal_at/bootstrap/cloudinit.py

@@ -1,4 +1,4 @@
-"""Render droplet cloud-init user-data (base packages + warmed toolchain + clones)."""
+"""Render host cloud-init user-data (base packages + warmed toolchain + clones)."""
 from __future__ import annotations
 
 from ..config import Config
@@ -10,4 +10,5 @@ def user_data(cfg: Config, index: int, name: str) -> str:  # noqa: ARG001 - sign
         "cloud_init.yaml.j2",
         flintlock_ref=cfg.flintlock_ref,
         brigade_ref=cfg.brigade_ref,
+        root_ssh_key=cfg.ssh_public_key if cfg.infra_backend == "libvirt" else "",
     )