Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions cmd/bbox/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ func rootCmd() *cobra.Command {
noSettings bool
noFirmwareDL bool
noImageCache bool
pull string
traceEnabled bool
timings bool
exec string
Expand Down Expand Up @@ -159,6 +160,7 @@ Example:
noSettings: noSettings,
noFirmwareDL: noFirmwareDL,
noImageCache: noImageCache,
pull: pull,
traceEnabled: traceEnabled || os.Getenv("BBOX_TRACE") == "1",
timings: timings,
exec: exec,
Expand Down Expand Up @@ -195,6 +197,7 @@ Example:
cmd.Flags().BoolVar(&noSettings, "no-settings", false, "Disable injecting host agent settings (rules, skills, etc.) into the VM")
cmd.Flags().BoolVar(&noFirmwareDL, "no-firmware-download", false, "Disable firmware download (use system libkrunfw only)")
cmd.Flags().BoolVar(&noImageCache, "no-image-cache", false, "Disable OCI image caching (fresh pull every run)")
cmd.Flags().StringVar(&pull, "pull", "", "Image pull policy: always, background, if-not-present, never (default: background)")
cmd.Flags().BoolVar(&traceEnabled, "trace", false, "Enable OpenTelemetry tracing (writes trace.json to VM data dir)")
cmd.Flags().BoolVar(&timings, "timings", false, "Print per-phase timing summary after run")
cmd.Flags().StringVar(&exec, "exec", "", "Override the agent command (e.g. /bin/bash for debugging)")
Expand Down Expand Up @@ -351,6 +354,7 @@ type runFlags struct {
noSettings bool
noFirmwareDL bool
noImageCache bool
pull string
traceEnabled bool
timings bool
exec string
Expand All @@ -368,6 +372,19 @@ func run(parentCtx context.Context, agentName string, flags runFlags) error {
return fmt.Errorf("invalid agent name: %w", err)
}

// Validate --pull flag early.
if flags.pull != "" && !domainconfig.IsValidPullPolicy(flags.pull) {
return fmt.Errorf("invalid --pull %q: valid values are %v",
flags.pull, domainconfig.ValidPullPolicies())
}

// --no-image-cache + --pull=never is a contradiction: "never" requires the
// cache to serve hits, but "no-image-cache" disables it entirely.
if flags.noImageCache && flags.pull == domainconfig.PullNever {
return fmt.Errorf("--no-image-cache and --pull=never are incompatible: "+
"pull policy %q requires a cache to serve hits", domainconfig.PullNever)
}

// Resolve workspace early so we can derive a deterministic VM name.
earlyWs := flags.workspace
if earlyWs == "" {
Expand Down Expand Up @@ -544,6 +561,7 @@ func run(parentCtx context.Context, agentName string, flags runFlags) error {
Defaults: cfg.Defaults,
AgentOverrides: cfg.Agents,
ExtraEgressHosts: configEgressHosts,
Image: cfg.Image,
MCP: cfg.MCP,
SettingsImport: cfg.SettingsImport,
}
Expand Down Expand Up @@ -840,6 +858,7 @@ func run(parentCtx context.Context, agentName string, flags runFlags) error {
LogLevel: logLevel,
CommandArgs: flags.commandArgs,
EnvForwardExtra: flags.envForward,
PullPolicy: flags.pull,
Snapshot: sandbox.SnapshotOpts{
Enabled: true,
SnapshotMatcher: snapshotMatcher,
Expand Down
12 changes: 12 additions & 0 deletions internal/infra/config/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,18 @@ var defaultConfigTemplate = `# Brood Box configuration
# # ports: [443]
# # protocol: 6 # TCP

# OCI image pulling behavior.
# image:
# # Image pull policy: always, background, if-not-present, never.
# # "always" — always check the registry for a new digest before starting;
# # still uses the digest-based cache so unchanged images are not re-extracted.
# # "background" — use the cached image instantly, check the registry in the
# # background; a newer image is cached for the next run (default).
# # "if-not-present" — use the cache if available, otherwise pull.
# # "never" — use the cache only; fail if the image is not cached.
# # Useful for airgapped/offline environments and CI.
# pull: "background"

# MCP (Model Context Protocol) proxy configuration.
# The MCP proxy discovers servers from ToolHive and makes them
# available to the agent inside the VM.
Expand Down
1 change: 1 addition & 0 deletions internal/infra/config/writer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ func TestWriteDefault(t *testing.T) {
content := string(data)
for _, section := range []string{
"defaults:", "review:", "network:",
"image:", "pull:",
"mcp:", "authz:", "config:",
"git:", "auth:", "settings_import:",
"runtime:", "agents:",
Expand Down
80 changes: 80 additions & 0 deletions internal/infra/vm/pullpolicy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
// SPDX-License-Identifier: Apache-2.0

package vm

import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"log/slog"
"os"
"path/filepath"
"time"

v1 "github.com/google/go-containerregistry/pkg/v1"

"github.com/stacklok/go-microvm/image"

"github.com/stacklok/brood-box/pkg/domain/config"
)

// backgroundRefreshTimeout is the maximum time allowed for a background
// image refresh. This covers the registry manifest fetch and, if the
// image changed, layer extraction and caching.
const backgroundRefreshTimeout = 5 * time.Minute

// neverPullFetcher implements image.ImageFetcher and always returns an error.
// Used with PullNever: if the cache ref index misses, this fetcher prevents
// any network access and returns a clear error message.
type neverPullFetcher struct{}

func (neverPullFetcher) Pull(_ context.Context, ref string) (v1.Image, error) {
return nil, fmt.Errorf("image %q not found in cache and pull policy is %q", ref, config.PullNever)
}

// deleteRefIndex removes the ref index entry for the given image reference
// from the OCI image cache. This forces the next pull to contact the registry
// for a fresh digest, while the digest-based cache still avoids re-extraction
// if the image hasn't changed.
//
// The ref index stores files at {cacheDir}/refs/{sha256(imageRef)}, matching
// the go-microvm image.Cache.refPath() convention.
func deleteRefIndex(cacheDir, imageRef string) error {
h := sha256.Sum256([]byte(imageRef))
refPath := filepath.Join(cacheDir, "refs", hex.EncodeToString(h[:]))
if err := os.Remove(refPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing ref index entry for %q: %w", imageRef, err)
}
return nil
}

// backgroundImageRefresh checks the registry for a newer image and caches
// it for the next run. The current VM continues using its existing rootfs.
// This is a fire-and-forget operation — errors are logged but do not
// affect the running session.
func backgroundImageRefresh(imageCacheDir, imageRef string, logger *slog.Logger) {
ctx, cancel := context.WithTimeout(context.Background(), backgroundRefreshTimeout)
defer cancel()

// Delete the ref index entry so PullWithFetcher contacts the registry
// instead of short-circuiting on the cached ref.
if err := deleteRefIndex(imageCacheDir, imageRef); err != nil {
logger.Debug("background image refresh: failed to clear ref index", "error", err)
return
}

cache := image.NewCache(imageCacheDir)
rootfs, err := image.PullWithFetcher(ctx, imageRef, cache, nil)
if err != nil {
logger.Debug("background image refresh failed", "image", imageRef, "error", err)
return
}

if rootfs.FromCache {
logger.Debug("background image refresh: image unchanged", "image", imageRef)
} else {
logger.Info("background image refresh: cached newer image for next run", "image", imageRef)
}
}
67 changes: 67 additions & 0 deletions internal/infra/vm/pullpolicy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
// SPDX-License-Identifier: Apache-2.0

package vm

import (
"context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/stacklok/brood-box/pkg/domain/config"
)

func TestNeverPullFetcher_ReturnsError(t *testing.T) {
t.Parallel()

f := neverPullFetcher{}
img, err := f.Pull(context.Background(), "ghcr.io/org/image:latest")

assert.Nil(t, img)
require.Error(t, err)
assert.Contains(t, err.Error(), "not found in cache")
assert.Contains(t, err.Error(), config.PullNever)
assert.Contains(t, err.Error(), "ghcr.io/org/image:latest")
}

func TestDeleteRefIndex_RemovesExistingRef(t *testing.T) {
t.Parallel()

cacheDir := t.TempDir()
refsDir := filepath.Join(cacheDir, "refs")
require.NoError(t, os.MkdirAll(refsDir, 0o700))

imageRef := "ghcr.io/org/image:latest"
h := sha256.Sum256([]byte(imageRef))
refFile := filepath.Join(refsDir, hex.EncodeToString(h[:]))
require.NoError(t, os.WriteFile(refFile, []byte(imageRef+"\tsha256:abc123\n"), 0o600))

err := deleteRefIndex(cacheDir, imageRef)
require.NoError(t, err)

_, statErr := os.Stat(refFile)
assert.True(t, os.IsNotExist(statErr))
}

func TestDeleteRefIndex_NoErrorWhenMissing(t *testing.T) {
t.Parallel()

cacheDir := t.TempDir()
err := deleteRefIndex(cacheDir, "ghcr.io/org/nonexistent:latest")
assert.NoError(t, err)
}

func TestDeleteRefIndex_NoErrorWhenRefsDirectoryMissing(t *testing.T) {
t.Parallel()

cacheDir := t.TempDir()
// Don't create refs/ subdirectory.
err := deleteRefIndex(cacheDir, "ghcr.io/org/image:latest")
assert.NoError(t, err)
}
25 changes: 25 additions & 0 deletions internal/infra/vm/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
microvmssh "github.com/stacklok/go-microvm/ssh"

"github.com/stacklok/brood-box/pkg/domain/agent"
"github.com/stacklok/brood-box/pkg/domain/config"
"github.com/stacklok/brood-box/pkg/domain/credential"
"github.com/stacklok/brood-box/pkg/domain/settings"
domvm "github.com/stacklok/brood-box/pkg/domain/vm"
Expand Down Expand Up @@ -306,6 +307,24 @@ func (r *MicroVMRunner) Start(ctx context.Context, cfg domvm.VMConfig) (domvm.VM
opts = append(opts, microvm.WithImageCache(image.NewCache(r.imageCacheDir)))
}

// Apply image pull policy.
switch cfg.PullPolicy {
case config.PullAlways:
// Delete the ref index entry so PullWithFetcher skips the fast
// path and contacts the registry for a fresh digest. The digest-
// based cache still avoids re-extraction for unchanged images.
if r.imageCacheDir != "" {
if err := deleteRefIndex(r.imageCacheDir, cfg.Image); err != nil {
r.logger.Warn("failed to clear ref index for --pull=always", "error", err)
}
}
case config.PullNever:
// Use a fetcher that always fails. If the ref index hits, the
// fetcher is never called. If it misses, the error is returned.
opts = append(opts, microvm.WithImageFetcher(neverPullFetcher{}))
}
// PullIfNotPresent (default): no additional options needed.

// Add workspace mount if specified.
if cfg.WorkspacePath != "" {
absPath, err := filepath.Abs(cfg.WorkspacePath)
Expand Down Expand Up @@ -342,6 +361,12 @@ func (r *MicroVMRunner) Start(ctx context.Context, cfg domvm.VMConfig) (domvm.VM
}
r.logger.Debug("sandbox VM started", "elapsed", time.Since(start))

// Background pull: refresh the image cache asynchronously so the
// next run picks up any newer image. The current VM is unaffected.
if cfg.PullPolicy == config.PullBackground && r.imageCacheDir != "" {
go backgroundImageRefresh(r.imageCacheDir, cfg.Image, r.logger)
}

return &microvmVM{
vm: pvm,
sshPort: sshPort,
Expand Down
51 changes: 51 additions & 0 deletions pkg/domain/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ type Config struct {
// Network configures egress networking.
Network NetworkConfig `yaml:"network"`

// Image configures OCI image pulling behavior.
Image ImageConfig `yaml:"image"`

// MCP configures the in-process MCP proxy.
MCP MCPConfig `yaml:"mcp"`

Expand Down Expand Up @@ -162,6 +165,49 @@ type RuntimeConfig struct {
FirmwareDownload *bool `yaml:"firmware_download,omitempty"`
}

// ImageConfig configures OCI image pulling behavior.
type ImageConfig struct {
// Pull controls the image pull policy.
// Valid values: "always", "if-not-present", "never".
// Default: "if-not-present".
Pull string `yaml:"pull,omitempty"`
}

const (
// PullAlways always checks the registry for a new digest before
// starting the VM. Still uses the digest-based cache — if the
// registry returns the same digest, the cached extraction is reused.
PullAlways = "always"

// PullBackground uses the cached image for the current run (instant
// start) and checks the registry in the background. If a newer image
// is found, it is cached for the next run. This is the default.
PullBackground = "background"

// PullIfNotPresent uses the cache if available, otherwise pulls from
// the registry.
PullIfNotPresent = "if-not-present"

// PullNever uses the cache only. Returns an error if the image is not
// cached. Useful for airgapped/offline environments and CI.
PullNever = "never"
)

// IsValidPullPolicy reports whether the given pull policy name is recognized.
func IsValidPullPolicy(policy string) bool {
switch policy {
case PullAlways, PullBackground, PullIfNotPresent, PullNever:
return true
default:
return false
}
}

// ValidPullPolicies returns the list of valid pull policy names.
func ValidPullPolicies() []string {
return []string{PullAlways, PullBackground, PullIfNotPresent, PullNever}
}

// GitTokenEnabled returns whether git token forwarding is enabled.
// Defaults to true when ForwardToken is nil.
func (g GitConfig) GitTokenEnabled() bool {
Expand Down Expand Up @@ -583,6 +629,11 @@ func MergeConfigs(global, local *Config) *Config {
// Runtime: local overrides global when explicitly set.
result.Runtime = mergeRuntimeConfig(global.Runtime, local.Runtime)

// Image: local overrides global when non-empty.
if local.Image.Pull != "" {
result.Image.Pull = local.Image.Pull
}

// MCP.Authz: local can only tighten (not widen). Same security pattern
// as egress profiles — a workspace config cannot escalate permissions.
result.MCP.Authz = MergeMCPAuthzConfig(global.MCP.Authz, local.MCP.Authz)
Expand Down
Loading