many: redo customizations in a bootc install to-filesystem world

This commit implements filesystem customizations for images that
got generated via `bootc install to-filesystem` via the new
`org.osbuild.bind` mechanism.

First the image is written via the `bootc.install-to-fs` stage,
then the image is mounted, the ostree.deployment stage is used
to get the right deployment and finally the mount is put over
the "tree" directory via the bind mount module. This way for
the users and selinux stages nothing changes, they work as
before on a normal tree (that just happens to be mounted from an
image this time).

It also includes a stage to make the /var/home directory that is
missing by default on the generic images.

pkg/image/bootc_disk.go

@@ -26,11 +26,13 @@ type BootcDiskImage struct {
 	// Customizations
 	KernelOptionsAppend []string
 
-	// "Users" is a bit misleading as only root and its ssh key is supported
-	// right now because that is all that bootc gives us by default but that
-	// will most likely change over time.
-	// See https://github.com/containers/bootc/pull/267
+	// The users to put into the image, note that /etc/paswd (and friends)
+	// will become unmanaged state by bootc when used
 	Users []users.User
+
+	// SELinux policy, when set it enables the labeling of the tree with the
+	// selected profile
+	SELinux string
 }
 
 func NewBootcDiskImage(container container.SourceSpec) *BootcDiskImage {
@@ -52,24 +54,23 @@ func (img *BootcDiskImage) InstantiateManifestFromContainers(m *manifest.Manifes
 	// this is signified by passing nil to the below pipelines.
 	var hostPipeline manifest.Build
 
-	// TODO: no support for customization right now but minimal support
-	// for root ssh keys is supported
-	baseImage := manifest.NewRawBootcImage(buildPipeline, containers, img.Platform)
-	baseImage.PartitionTable = img.PartitionTable
-	baseImage.Users = img.Users
-	baseImage.KernelOptionsAppend = img.KernelOptionsAppend
+	rawImage := manifest.NewRawBootcImage(buildPipeline, containers, img.Platform)
+	rawImage.PartitionTable = img.PartitionTable
+	rawImage.Users = img.Users
+	rawImage.KernelOptionsAppend = img.KernelOptionsAppend
+	rawImage.SELinux = img.SELinux
 
 	// In BIB, we export multiple images from the same pipeline so we use the
 	// filename as the basename for each export and set the extensions based on
 	// each file format.
 	fileBasename := img.Filename
-	baseImage.SetFilename(fmt.Sprintf("%s.raw", fileBasename))
+	rawImage.SetFilename(fmt.Sprintf("%s.raw", fileBasename))
 
-	qcow2Pipeline := manifest.NewQCOW2(hostPipeline, baseImage)
+	qcow2Pipeline := manifest.NewQCOW2(hostPipeline, rawImage)
 	qcow2Pipeline.Compat = img.Platform.GetQCOW2Compat()
 	qcow2Pipeline.SetFilename(fmt.Sprintf("%s.qcow2", fileBasename))
 
-	vmdkPipeline := manifest.NewVMDK(hostPipeline, baseImage)
+	vmdkPipeline := manifest.NewVMDK(hostPipeline, rawImage)
 	vmdkPipeline.SetFilename(fmt.Sprintf("%s.vmdk", fileBasename))
 
 	ovfPipeline := manifest.NewOVF(hostPipeline, vmdkPipeline)

pkg/image/bootc_disk_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/osbuild/images/internal/testdisk"
 	"github.com/osbuild/images/pkg/container"
+	"github.com/osbuild/images/pkg/customizations/users"
 	"github.com/osbuild/images/pkg/image"
 	"github.com/osbuild/images/pkg/manifest"
 	"github.com/osbuild/images/pkg/platform"
@@ -38,6 +39,8 @@ func makeFakeDigest(t *testing.T) string {
 type bootcDiskImageTestOpts struct {
 	ImageFormat platform.ImageFormat
 	BIOS        bool
+	SELinux     string
+	Users       []users.User
 
 	KernelOptionsAppend []string
 }
@@ -70,6 +73,8 @@ func makeBootcDiskImageOsbuildManifest(t *testing.T, opts *bootcDiskImageTestOpt
 	img.Platform = makeFakePlatform(opts)
 	img.PartitionTable = testdisk.MakeFakePartitionTable("/", "/boot", "/boot/efi")
 	img.KernelOptionsAppend = opts.KernelOptionsAppend
+	img.Users = opts.Users
+	img.SELinux = opts.SELinux
 
 	m := &manifest.Manifest{}
 	runi := &runner.Fedora{}
@@ -182,3 +187,42 @@ func TestBootcDiskImageExportPipelines(t *testing.T) {
 	tarPipeline := findPipelineFromOsbuildManifest(t, osbuildManifest, "archive")
 	require.NotNil(tarPipeline)
 }
+
+func TestBootcDiskImageInstantiateUsers(t *testing.T) {
+	for _, withUsers := range []bool{true, false} {
+		opts := &bootcDiskImageTestOpts{}
+		if withUsers {
+			opts.Users = []users.User{{Name: "foo"}}
+		}
+		osbuildManifest := makeBootcDiskImageOsbuildManifest(t, opts)
+		imagePipeline := findPipelineFromOsbuildManifest(t, osbuildManifest, "image")
+		require.NotNil(t, imagePipeline)
+		usersStage := findStageFromOsbuildPipeline(t, imagePipeline, "org.osbuild.users")
+		if withUsers {
+			require.NotNil(t, usersStage)
+		} else {
+			require.Nil(t, usersStage)
+		}
+	}
+}
+
+func TestBootcDiskImageInstantiateSELinuxForUsers(t *testing.T) {
+	for _, withSELinux := range []string{"", "targeted"} {
+		opts := &bootcDiskImageTestOpts{
+			Users: []users.User{
+				{Name: "foo"},
+			},
+			SELinux: withSELinux,
+		}
+		osbuildManifest := makeBootcDiskImageOsbuildManifest(t, opts)
+
+		imagePipeline := findPipelineFromOsbuildManifest(t, osbuildManifest, "image")
+		require.NotNil(t, imagePipeline)
+		selinuxStage := findStageFromOsbuildPipeline(t, imagePipeline, "org.osbuild.selinux")
+		if withSELinux != "" {
+			require.NotNil(t, selinuxStage)
+		} else {
+			require.Nil(t, selinuxStage)
+		}
+	}
+}

pkg/manifest/raw_bootc.go

@@ -2,7 +2,9 @@ package manifest
 
 import (
 	"fmt"
+	"os"
 
+	"github.com/osbuild/images/internal/common"
 	"github.com/osbuild/images/pkg/artifact"
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/customizations/users"
@@ -31,11 +33,13 @@ type RawBootcImage struct {
 
 	KernelOptionsAppend []string
 
-	// "Users" is a bit misleading as only root and its ssh key is supported
-	// right now because that is all that bootc gives us by default but that
-	// will most likely change over time.
-	// See https://github.com/containers/bootc/pull/267
+	// The users to put into the image, note that /etc/paswd (and friends)
+	// will become unmanaged state by bootc when used
 	Users []users.User
+
+	// SELinux policy, when set it enables the labeling of the tree with the
+	// selected profile
+	SELinux string
 }
 
 func (p RawBootcImage) Filename() string {
@@ -88,13 +92,6 @@ func (p *RawBootcImage) serialize() osbuild.Pipeline {
 		panic(fmt.Errorf("no partition table in live image"))
 	}
 
-	if len(p.Users) > 1 {
-		panic(fmt.Errorf("raw bootc image only supports a single root key for user customization, got %v", p.Users))
-	}
-	if len(p.Users) == 1 && p.Users[0].Name != "root" {
-		panic(fmt.Errorf("raw bootc image only supports the root user, got %v", p.Users))
-	}
-
 	for _, stage := range osbuild.GenImagePrepareStages(pt, p.filename, osbuild.PTSfdisk) {
 		pipeline.AddStage(stage)
 	}
@@ -105,9 +102,6 @@ func (p *RawBootcImage) serialize() osbuild.Pipeline {
 	opts := &osbuild.BootcInstallToFilesystemOptions{
 		Kargs: p.KernelOptionsAppend,
 	}
-	if len(p.Users) == 1 && p.Users[0].Key != nil {
-		opts.RootSSHAuthorizedKeys = []string{*p.Users[0].Key}
-	}
 	inputs := osbuild.ContainerDeployInputs{
 		Images: osbuild.NewContainersInputForSingleSource(p.containerSpecs[0]),
 	}
@@ -121,17 +115,56 @@ func (p *RawBootcImage) serialize() osbuild.Pipeline {
 	}
 	pipeline.AddStage(st)
 
-	// XXX: there is no way right now to support any customizations,
-	// we cannot touch the filesystem after bootc installed it or
-	// we risk messing with it's selinux labels or future fsverity
-	// magic.  Once we have a mechanism like --copy-etc from
-	// https://github.com/containers/bootc/pull/267 things should
-	// be a bit better
-
 	for _, stage := range osbuild.GenImageFinishStages(pt, p.filename) {
 		pipeline.AddStage(stage)
 	}
 
+	// customize the image
+	if len(p.Users) > 0 {
+		// build common devices/mounts first
+		devices, mounts, err := osbuild.GenBootupdDevicesMounts(p.filename, p.PartitionTable)
+		if err != nil {
+			panic(fmt.Sprintf("gen devices stage failed %v", err))
+		}
+		mounts = append(mounts, *osbuild.NewOSTreeDeploymentMountDefault("ostree.deployment", osbuild.OSTreeMountSourceMount))
+		mounts = append(mounts, *osbuild.NewBindMount("bind-ostree-deployment-to-tree", "mount://", "tree://"))
+
+		// ensure /var/home is available
+		mkdirStage := osbuild.NewMkdirStage(&osbuild.MkdirStageOptions{
+			Paths: []osbuild.MkdirStagePath{
+				{
+					Path:    "/var/home",
+					Mode:    common.ToPtr(os.FileMode(0755)),
+					ExistOk: true,
+				},
+			},
+		})
+		mkdirStage.Mounts = mounts
+		mkdirStage.Devices = devices
+		pipeline.AddStage(mkdirStage)
+
+		// add the users
+		usersStage, err := osbuild.GenUsersStage(p.Users, false)
+		if err != nil {
+			panic(fmt.Sprintf("user stage failed %v", err))
+		}
+		usersStage.Mounts = mounts
+		usersStage.Devices = devices
+		pipeline.AddStage(usersStage)
+
+		// add selinux
+		if p.SELinux != "" {
+			opts := &osbuild.SELinuxStageOptions{
+				FileContexts: fmt.Sprintf("etc/selinux/%s/contexts/files/file_contexts", p.SELinux),
+				ExcludePaths: []string{"/sysroot"},
+			}
+			selinuxStage := osbuild.NewSELinuxStage(opts)
+			selinuxStage.Mounts = mounts
+			selinuxStage.Devices = devices
+			pipeline.AddStage(selinuxStage)
+		}
+	}
+
 	return pipeline
 }
 

pkg/manifest/raw_bootc_test.go

@@ -1,13 +1,11 @@
 package manifest_test
 
 import (
-	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/osbuild/images/internal/assertx"
 	"github.com/osbuild/images/internal/common"
 	"github.com/osbuild/images/internal/testdisk"
 	"github.com/osbuild/images/pkg/container"
@@ -58,7 +56,9 @@ func TestRawBootcImageSerialize(t *testing.T) {
 	bootcInst := manifest.FindStage("org.osbuild.bootc.install-to-filesystem", imagePipeline.Stages)
 	require.NotNil(t, bootcInst)
 	opts := bootcInst.Options.(*osbuild.BootcInstallToFilesystemOptions)
-	assert.Equal(t, []string{"some-ssh-key"}, opts.RootSSHAuthorizedKeys)
+	// Note that the root account is customized via the "users" stage
+	// (mostly for uniformity)
+	assert.Equal(t, len(opts.RootSSHAuthorizedKeys), 0)
 	assert.Equal(t, []string{"karg1", "karg2"}, opts.Kargs)
 }
 
@@ -76,38 +76,87 @@ func TestRawBootcImageSerializeMountsValidated(t *testing.T) {
 	})
 }
 
-func TestRawBootcImageSerializeValidatesUsers(t *testing.T) {
+func findMountIdx(mounts []osbuild.Mount, mntType string) int {
+	for i, mnt := range mounts {
+		if mnt.Type == mntType {
+			return i
+		}
+	}
+	return -1
+}
+
+func makeFakeRawBootcPipeline() *manifest.RawBootcImage {
 	mani := manifest.New()
 	runner := &runner.Linux{}
 	build := manifest.NewBuildFromContainer(&mani, runner, nil, nil)
-
 	rawBootcPipeline := manifest.NewRawBootcImage(build, nil, nil)
 	rawBootcPipeline.PartitionTable = testdisk.MakeFakePartitionTable("/", "/boot", "/boot/efi")
 	rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil)
 
+	return rawBootcPipeline
+}
+
+func TestRawBootcImageSerializeCreateUsersOptions(t *testing.T) {
+	rawBootcPipeline := makeFakeRawBootcPipeline()
+
 	for _, tc := range []struct {
-		users       []users.User
-		expectedErr string
+		users              []users.User
+		expectedUsersStage bool
 	}{
-		// good
-		{nil, ""},
-		{[]users.User{{Name: "root"}}, ""},
-		{[]users.User{{Name: "root", Key: common.ToPtr("some-key")}}, ""},
-		// bad
-		{[]users.User{{Name: "foo"}},
-			"raw bootc image only supports the root user, got.*"},
-		{[]users.User{{Name: "root"}, {Name: "foo"}},
-			"raw bootc image only supports a single root key for user customization, got.*"},
+		{nil, false},
+		{[]users.User{{Name: "root"}}, true},
+		{[]users.User{{Name: "foo"}}, true},
+		{[]users.User{{Name: "root"}, {Name: "foo"}}, true},
 	} {
 		rawBootcPipeline.Users = tc.users
 
-		if tc.expectedErr == "" {
-			rawBootcPipeline.Serialize()
+		pipeline := rawBootcPipeline.Serialize()
+		usersStage := manifest.FindStage("org.osbuild.users", pipeline.Stages)
+		if tc.expectedUsersStage {
+			// ensure options got passed
+			require.NotNil(t, usersStage)
+			userOptions := usersStage.Options.(*osbuild.UsersStageOptions)
+			for _, user := range tc.users {
+				assert.NotNil(t, userOptions.Users[user.Name])
+			}
 		} else {
-			expectedErr := regexp.MustCompile(tc.expectedErr)
-			assertx.PanicsWithErrorRegexp(t, expectedErr, func() {
-				rawBootcPipeline.Serialize()
-			})
+			require.Nil(t, usersStage)
+		}
+	}
+}
+
+func assertBootcDeploymentAndBindMount(t *testing.T, stage *osbuild.Stage) {
+	// check for bind mount to deployment is there so
+	// that the customization actually works
+	deploymentMntIdx := findMountIdx(stage.Mounts, "org.osbuild.ostree.deployment")
+	assert.True(t, deploymentMntIdx >= 0)
+	bindMntIdx := findMountIdx(stage.Mounts, "org.osbuild.bind")
+	assert.True(t, bindMntIdx >= 0)
+	// order is important, bind must happen *after* deploy
+	assert.True(t, bindMntIdx > deploymentMntIdx)
+}
+
+func TestRawBootcImageSerializeCustomizationGenCorrectStages(t *testing.T) {
+	rawBootcPipeline := makeFakeRawBootcPipeline()
+
+	for _, tc := range []struct {
+		users   []users.User
+		SELinux string
+
+		expectedStages []string
+	}{
+		{nil, "", nil},
+		{[]users.User{{Name: "foo"}}, "", []string{"org.osbuild.mkdir", "org.osbuild.users"}},
+		{[]users.User{{Name: "foo"}}, "targeted", []string{"org.osbuild.mkdir", "org.osbuild.users", "org.osbuild.selinux"}},
+	} {
+		rawBootcPipeline.Users = tc.users
+		rawBootcPipeline.SELinux = tc.SELinux
+
+		pipeline := rawBootcPipeline.Serialize()
+		for _, expectedStage := range tc.expectedStages {
+			stage := manifest.FindStage(expectedStage, pipeline.Stages)
+			assert.NotNil(t, stage)
+			assertBootcDeploymentAndBindMount(t, stage)
 		}
 	}
 }