Remove discovery stateroot when installing to existing root (#7283)

When installing to the existing root filesystem the host finishes
install with twice the data in the /sysroot and /boot filesystems as it
should have.

For /sysroot this isn't much of a problem, but /boot ends up getting to
~96% utilized during upgrades with the extra kernel and initrd in there.

To handle this a script and a oneshot service to run it is added to the
host pointer ignition when the host is being installed to an existing
root. This script undeploys the boot entry which removes the files from
/boot and removes the stateroot from /sysroot. This service only runs on
the first boot of the machine.

Resolves https://issues.redhat.com/browse/MGMT-19106

internal/ignition/installmanifests.go

@@ -60,6 +60,44 @@ const highlyAvailableInfrastructureTopologyPatch = `---
   value: HighlyAvailable
 `
 
+/*
+Contents of the base64 encoded file are:
+
+#!/bin/bash
+
+set -eux
+unshare --mount
+mount -oremount,rw /sysroot
+ostree admin undeploy 1
+rm -rf /sysroot/ostree/deploy/rhcos
+*/
+const cleanupDiscoveryStaterootIgnitionOverride = `{
+  "ignition": {
+    "version": "3.2.0"
+  },
+  "storage": {
+    "files": [{
+      "overwrite": true,
+      "path": "/usr/local/bin/cleanup-assisted-discovery-stateroot.sh",
+      "mode": 493,
+      "user": {
+          "name": "root"
+      },
+      "contents": { "source": "data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCnNldCAtZXV4CnVuc2hhcmUgLS1tb3VudAptb3VudCAtb3JlbW91bnQscncgL3N5c3Jvb3QKb3N0cmVlIGFkbWluIHVuZGVwbG95IDEKcm0gLXJmIC9zeXNyb290L29zdHJlZS9kZXBsb3kvcmhjb3MK" }
+    }]
+  },
+  "systemd": {
+    "units": [
+      {
+        "contents": "[Unit]\nDescription=Cleanup Assisted Installer discovery stateroot\nConditionFirstBoot=yes\nConditionPathExists=/sysroot/ostree/deploy/rhcos\nBefore=first-boot-complete.target\nWants=first-boot-complete.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/local/bin/cleanup-assisted-discovery-stateroot.sh\n\n[Install]\nWantedBy=basic.target\n",
+        "enabled": true,
+        "name": "cleanup-assisted-discovery-stateroot.service"
+      }
+    ]
+  }
+}
+`
+
 type clusterVersion struct {
 	APIVersion string `yaml:"apiVersion"`
 	Metadata   struct {
@@ -1135,6 +1173,21 @@ func (g *installerGenerator) writeSingleHostFile(host *models.Host, baseFile str
 		configBytes = []byte(override)
 	}
 
+	inventory := models.Inventory{}
+	if host.Inventory != "" {
+		if err = json.Unmarshal([]byte(host.Inventory), &inventory); err != nil {
+			return err
+		}
+	}
+	if inventory.Boot != nil && inventory.Boot.DeviceType == models.BootDeviceTypePersistent {
+		g.log.Infof("Adding stateroot cleanup ignition override for host %s", host.ID)
+		merged, mergeErr := MergeIgnitionConfig(configBytes, []byte(cleanupDiscoveryStaterootIgnitionOverride))
+		if mergeErr != nil {
+			return errors.Wrapf(mergeErr, "failed to apply stateroot cleanup ignition override for host %s", host.ID)
+		}
+		configBytes = []byte(merged)
+	}
+
 	err = os.WriteFile(filepath.Join(workDir, hostutil.IgnitionFileName(host)), configBytes, 0600)
 	if err != nil {
 		return errors.Wrapf(err, "failed to write ignition for host %s", host.ID)

internal/ignition/installmanifests_test.go

@@ -839,6 +839,45 @@ var _ = Describe("createHostIgnitions", func() {
 
 		Expect(*exampleFile.FileEmbedded1.Contents.Source).To(Equal("data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"))
 	})
+
+	It("writes the stateroot cleanup overrides when boot device is persistent", func() {
+		hostID := strfmt.UUID(uuid.New().String())
+		cluster.Hosts = []*models.Host{{
+			ID:                &hostID,
+			RequestedHostname: "master0.example.com",
+			Role:              models.HostRoleMaster,
+			Inventory:         `{"boot": {"device_type": "persistent"}}`,
+		}}
+
+		g := NewGenerator(workDir, cluster, "", "", "", "", nil, logrus.New(), nil, "", "", manifestsAPI, eventsHandler, installerCache).(*installerGenerator)
+
+		err := g.createHostIgnitions()
+		Expect(err).NotTo(HaveOccurred())
+
+		ignBytes, err := os.ReadFile(filepath.Join(workDir, fmt.Sprintf("%s-%s.ign", models.HostRoleMaster, hostID)))
+		Expect(err).NotTo(HaveOccurred())
+		config, _, err := config_32.Parse(ignBytes)
+		Expect(err).NotTo(HaveOccurred())
+
+		var scriptFile *config_32_types.File
+		for idx, file := range config.Storage.Files {
+			if file.Node.Path == "/usr/local/bin/cleanup-assisted-discovery-stateroot.sh" {
+				scriptFile = &config.Storage.Files[idx]
+			}
+		}
+		Expect(scriptFile).NotTo(BeNil())
+		Expect(*scriptFile.FileEmbedded1.Contents.Source).To(Equal("data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCnNldCAtZXV4CnVuc2hhcmUgLS1tb3VudAptb3VudCAtb3JlbW91bnQscncgL3N5c3Jvb3QKb3N0cmVlIGFkbWluIHVuZGVwbG95IDEKcm0gLXJmIC9zeXNyb290L29zdHJlZS9kZXBsb3kvcmhjb3MK"))
+
+		var unit *config_32_types.Unit
+		for idx, u := range config.Systemd.Units {
+			if u.Name == "cleanup-assisted-discovery-stateroot.service" {
+				unit = &config.Systemd.Units[idx]
+			}
+		}
+		Expect(unit).NotTo(BeNil())
+		Expect(*unit.Contents).To(Equal("[Unit]\nDescription=Cleanup Assisted Installer discovery stateroot\nConditionFirstBoot=yes\nConditionPathExists=/sysroot/ostree/deploy/rhcos\nBefore=first-boot-complete.target\nWants=first-boot-complete.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/local/bin/cleanup-assisted-discovery-stateroot.sh\n\n[Install]\nWantedBy=basic.target\n"))
+	})
+
 	Context("machine config pool", func() {
 		const (
 			mcp = `apiVersion: machineconfiguration.openshift.io/v1