Support cloud config secrets for templates and backup/restores

pkg/api/vm/handler.go

@@ -17,6 +17,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/client-go/rest"
 	kv1 "kubevirt.io/client-go/api/v1"
 
@@ -49,6 +50,8 @@ type vmActionHandler struct {
 	settingCache              ctlharvesterv1.SettingCache
 	nodeCache                 ctlcorev1.NodeCache
 	pvcCache                  ctlcorev1.PersistentVolumeClaimCache
+	secretClient              ctlcorev1.SecretClient
+	secretCache               ctlcorev1.SecretCache
 	virtSubresourceRestClient rest.Interface
 	virtRestClient            rest.Interface
 }
@@ -418,22 +421,74 @@ func (h *vmActionHandler) createTemplate(namespace, name string, input CreateTem
 	if err != nil {
 		return err
 	}
+
 	vmID := fmt.Sprintf("%s/%s", vmt.Namespace, vmt.Name)
 
-	_, err = h.vmTemplateVersionClient.Create(
+	vmtvName := fmt.Sprintf("%s-%s", vmt.Name, rand.String(5))
+	vmtv, err := h.vmTemplateVersionClient.Create(
 		&harvesterv1.VirtualMachineTemplateVersion{
 			ObjectMeta: metav1.ObjectMeta{
-				GenerateName: fmt.Sprintf("%s-", vmt.Name),
-				Namespace:    namespace,
+				Name:      vmtvName,
+				Namespace: namespace,
 			},
 			Spec: harvesterv1.VirtualMachineTemplateVersionSpec{
 				TemplateID:  vmID,
 				Description: fmt.Sprintf("Template drived from virtual machine [%s]", vmID),
-				VM:          removeMacAddresses(vm),
+				VM:          sanitizeVirtualMachineForTemplateVersion(vmtvName, vm),
 				KeyPairIDs:  keyPairIDs,
 			},
 		})
+	if err != nil {
+		return err
+	}
+
+	return h.createCloudConfigSecrets(vmtv, vm)
+}
+
+func (h *vmActionHandler) createCloudConfigSecrets(templateVersion *harvesterv1.VirtualMachineTemplateVersion, vm *kv1.VirtualMachine) error {
+	for _, volume := range vm.Spec.Template.Spec.Volumes {
+		if volume.CloudInitNoCloud == nil {
+			continue
+		}
+		if volume.CloudInitNoCloud.UserDataSecretRef != nil {
+			toCreateSecretName := getTemplateVersionUserDataSecretName(templateVersion.Name, volume.Name)
+			if err := h.copySecret(volume.CloudInitNoCloud.UserDataSecretRef.Name, toCreateSecretName, templateVersion); err != nil {
+				return err
+			}
+		}
+		if volume.CloudInitNoCloud.NetworkDataSecretRef != nil {
+			toCreateSecretName := getTemplateVersionNetworkDataSecretName(templateVersion.Name, volume.Name)
+			if err := h.copySecret(volume.CloudInitNoCloud.NetworkDataSecretRef.Name, toCreateSecretName, templateVersion); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (h *vmActionHandler) copySecret(sourceName, targetName string, templateVersion *harvesterv1.VirtualMachineTemplateVersion) error {
+	secret, err := h.secretCache.Get(templateVersion.Namespace, sourceName)
+	if err != nil {
+		return err
+	}
+	toCreate := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      targetName,
+			Namespace: secret.Namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: templateVersion.APIVersion,
+					Kind:       templateVersion.Kind,
+					Name:       templateVersion.Name,
+					UID:        templateVersion.UID,
+				},
+			},
+		},
+		Data: secret.Data,
+	}
+	_, err = h.secretClient.Create(toCreate)
 	return err
+
 }
 
 // addVolume add a hotplug volume with given volume source and disk name.
@@ -518,17 +573,40 @@ func (h *vmActionHandler) removeVolume(ctx context.Context, namespace, name stri
 		Error()
 }
 
+func sanitizeVirtualMachineForTemplateVersion(templateVersionName string, vm *kv1.VirtualMachine) harvesterv1.VirtualMachineSourceSpec {
+	sanitizedVm := removeMacAddresses(vm)
+	sanitizedVm = replaceCloudInitSecrets(templateVersionName, sanitizedVm)
+
+	return harvesterv1.VirtualMachineSourceSpec{
+		ObjectMeta: sanitizedVm.ObjectMeta,
+		Spec:       sanitizedVm.Spec,
+	}
+}
+
+func replaceCloudInitSecrets(templateVersionName string, vm *kv1.VirtualMachine) *kv1.VirtualMachine {
+	sanitizedVm := vm.DeepCopy()
+	for index, volume := range sanitizedVm.Spec.Template.Spec.Volumes {
+		if volume.CloudInitNoCloud == nil {
+			continue
+		}
+		if volume.CloudInitNoCloud.UserDataSecretRef != nil {
+			sanitizedVm.Spec.Template.Spec.Volumes[index].CloudInitNoCloud.UserDataSecretRef.Name = getTemplateVersionUserDataSecretName(templateVersionName, volume.Name)
+		}
+		if volume.CloudInitNoCloud.NetworkDataSecretRef != nil {
+			sanitizedVm.Spec.Template.Spec.Volumes[index].CloudInitNoCloud.NetworkDataSecretRef.Name = getTemplateVersionNetworkDataSecretName(templateVersionName, volume.Name)
+		}
+	}
+	return sanitizedVm
+}
+
 // removeMacAddresses replaces the mac address of each device interface with an empty string.
 // This is because macAddresses are unique, and should not reuse the original's.
-func removeMacAddresses(vm *kv1.VirtualMachine) harvesterv1.VirtualMachineSourceSpec {
+func removeMacAddresses(vm *kv1.VirtualMachine) *kv1.VirtualMachine {
 	sanitizedVm := vm.DeepCopy()
 	for index := range sanitizedVm.Spec.Template.Spec.Domain.Devices.Interfaces {
 		sanitizedVm.Spec.Template.Spec.Domain.Devices.Interfaces[index].MacAddress = ""
 	}
-	return harvesterv1.VirtualMachineSourceSpec{
-		ObjectMeta: sanitizedVm.ObjectMeta,
-		Spec:       sanitizedVm.Spec,
-	}
+	return sanitizedVm
 }
 
 // getSSHKeysFromVMITemplateSpec first checks the given VirtualMachineInstanceTemplateSpec
@@ -548,3 +626,11 @@ func getSSHKeysFromVMITemplateSpec(vmitSpec *kv1.VirtualMachineInstanceTemplateS
 	}
 	return sshKeys, nil
 }
+
+func getTemplateVersionUserDataSecretName(templateVersionName, volumeName string) string {
+	return fmt.Sprintf("templateversion-%s-%s-userdata", templateVersionName, volumeName)
+}
+
+func getTemplateVersionNetworkDataSecretName(templateVersionName, volumeName string) string {
+	return fmt.Sprintf("templateversion-%s-%s-networkdata", templateVersionName, volumeName)
+}

pkg/api/vm/schema.go

@@ -44,6 +44,7 @@ func RegisterSchema(scaled *config.Scaled, server *server.Server, options config
 	settings := scaled.HarvesterFactory.Harvesterhci().V1beta1().Setting()
 	nodes := scaled.CoreFactory.Core().V1().Node()
 	pvcs := scaled.CoreFactory.Core().V1().PersistentVolumeClaim()
+	secrets := scaled.CoreFactory.Core().V1().Secret()
 	vmt := scaled.HarvesterFactory.Harvesterhci().V1beta1().VirtualMachineTemplate()
 	vmtv := scaled.HarvesterFactory.Harvesterhci().V1beta1().VirtualMachineTemplateVersion()
 
@@ -75,6 +76,8 @@ func RegisterSchema(scaled *config.Scaled, server *server.Server, options config
 		settingCache:              settings.Cache(),
 		nodeCache:                 nodes.Cache(),
 		pvcCache:                  pvcs.Cache(),
+		secretClient:              secrets,
+		secretCache:               secrets.Cache(),
 		virtSubresourceRestClient: virtSubresourceClient,
 		virtRestClient:            virtv1Client.RESTClient(),
 	}

pkg/apis/harvesterhci.io/v1beta1/backup.go

@@ -69,6 +69,9 @@ type VirtualMachineBackupStatus struct {
 	// +optional
 	VolumeBackups []VolumeBackup `json:"volumeBackups,omitempty"`
 
+	// +optional
+	SecretBackups []SecretBackup `json:"secretBackups,omitempty"`
+
 	// +optional
 	ReadyToUse *bool `json:"readyToUse,omitempty"`
 
@@ -109,6 +112,15 @@ type VolumeBackup struct {
 	Error *Error `json:"error,omitempty"`
 }
 
+// SecretBackup contains the secret data need to restore a secret referenced by the VM
+type SecretBackup struct {
+	// +kubebuilder:validation:Required
+	Name string `json:"name,omitempty"`
+
+	// +optional
+	Data map[string][]byte `json:"data,omitempty"`
+}
+
 type PersistentVolumeClaimSourceSpec struct {
 	// +kubebuilder:pruning:PreserveUnknownFields
 	// +optional

pkg/controller/master/backup/backup.go

@@ -44,6 +44,7 @@ var vmBackupKind = harvesterv1.SchemeGroupVersion.WithKind(vmBackupKindName)
 func RegisterBackup(ctx context.Context, management *config.Management, opts config.Options) error {
 	vmBackups := management.HarvesterFactory.Harvesterhci().V1beta1().VirtualMachineBackup()
 	pvc := management.CoreFactory.Core().V1().PersistentVolumeClaim()
+	secrets := management.CoreFactory.Core().V1().Secret()
 	vms := management.VirtFactory.Kubevirt().V1().VirtualMachine()
 	volumes := management.LonghornFactory.Longhorn().V1beta1().Volume()
 	snapshots := management.SnapshotFactory.Snapshot().V1beta1().VolumeSnapshot()
@@ -54,6 +55,7 @@ func RegisterBackup(ctx context.Context, management *config.Management, opts con
 		vmBackupController: vmBackups,
 		vmBackupCache:      vmBackups.Cache(),
 		pvcCache:           pvc.Cache(),
+		secretCache:        secrets.Cache(),
 		vms:                vms,
 		vmsCache:           vms.Cache(),
 		volumeCache:        volumes.Cache(),
@@ -76,6 +78,7 @@ type Handler struct {
 	vms                ctlkubevirtv1.VirtualMachineClient
 	vmsCache           ctlkubevirtv1.VirtualMachineCache
 	pvcCache           ctlcorev1.PersistentVolumeClaimCache
+	secretCache        ctlcorev1.SecretCache
 	volumeCache        ctllonghornv1.VolumeCache
 	volumes            ctllonghornv1.VolumeClient
 	snapshots          ctlsnapshotv1.VolumeSnapshotClient
@@ -175,6 +178,36 @@ func (h *Handler) getVolumeBackups(backup *harvesterv1.VirtualMachineBackup, vm
 	return volumeBackups, nil
 }
 
+// getSecretBackups helps to build a list of SecretBackup upon the cloud init secrets used by the backup VM
+func (h *Handler) getSecretBackups(vm *kv1.VirtualMachine) ([]harvesterv1.SecretBackup, error) {
+	var secretBackups []harvesterv1.SecretBackup
+
+	for _, volume := range vm.Spec.Template.Spec.Volumes {
+		if volume.CloudInitNoCloud != nil && volume.CloudInitNoCloud.UserDataSecretRef != nil {
+			secret, err := h.secretCache.Get(vm.Namespace, volume.CloudInitNoCloud.UserDataSecretRef.Name)
+			if err != nil {
+				return nil, err
+			}
+			secretBackups = append(secretBackups, harvesterv1.SecretBackup{
+				Name: secret.Name,
+				Data: secret.Data,
+			})
+		}
+		if volume.CloudInitNoCloud != nil && volume.CloudInitNoCloud.NetworkDataSecretRef != nil {
+			secret, err := h.secretCache.Get(vm.Namespace, volume.CloudInitNoCloud.NetworkDataSecretRef.Name)
+			if err != nil {
+				return nil, err
+			}
+			secretBackups = append(secretBackups, harvesterv1.SecretBackup{
+				Name: secret.Name,
+				Data: secret.Data,
+			})
+		}
+	}
+
+	return secretBackups, nil
+}
+
 // updateBackupStatusContent helps to store the backup source and volume contents within the VM backup status
 func (h *Handler) updateBackupStatusContent(backup *harvesterv1.VirtualMachineBackup, vm *kv1.VirtualMachine) (*harvesterv1.VirtualMachineBackupStatus, error) {
 	var err error
@@ -187,6 +220,13 @@ func (h *Handler) updateBackupStatusContent(backup *harvesterv1.VirtualMachineBa
 		}
 	}
 
+	if status.SecretBackups == nil {
+		status.SecretBackups, err = h.getSecretBackups(vm)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if status.SourceSpec == nil {
 		status.SourceSpec = &harvesterv1.VirtualMachineSourceSpec{
 			ObjectMeta: metav1.ObjectMeta{