From abc91f0304f580185153c3667087d5f78f271153 Mon Sep 17 00:00:00 2001 From: futuretea Date: Mon, 21 Jun 2021 12:53:21 +0800 Subject: [PATCH] Refactor vmbuilder Signed-off-by: futuretea --- pkg/builder/cloudinit.go | 83 ++++ pkg/builder/disk.go | 208 ++++++++++ pkg/builder/network.go | 70 ++++ pkg/builder/vm.go | 227 +++++++++++ tests/integration/api/network_apis_test.go | 25 +- tests/integration/api/vm_apis_test.go | 50 ++- tests/integration/api/vm_apis_utils_test.go | 355 +----------------- .../integration/api/vm_backup_restore_test.go | 14 +- .../integration/api/vm_template_apis_test.go | 4 +- 9 files changed, 663 insertions(+), 373 deletions(-) create mode 100644 pkg/builder/cloudinit.go create mode 100644 pkg/builder/disk.go create mode 100644 pkg/builder/network.go create mode 100644 pkg/builder/vm.go diff --git a/pkg/builder/cloudinit.go b/pkg/builder/cloudinit.go new file mode 100644 index 0000000000..8f9ae66f5d --- /dev/null +++ b/pkg/builder/cloudinit.go @@ -0,0 +1,83 @@ +package builder + +import ( + corev1 "k8s.io/api/core/v1" + kubevirtv1 "kubevirt.io/client-go/api/v1" +) + +const ( + CloudInitTypeNoCloud = "noCloud" + CloudInitTypeConfigDrive = "configDrive" + CloudInitDiskName = "cloudinitdisk" +) + +type CloudInitSource struct { + CloudInitType string + UserDataSecretName string + UserDataBase64 string + UserData string + NetworkDataSecretName string + NetworkDataBase64 string + NetworkData string +} + +func (v *VMBuilder) CloudInit(diskName string, cloudInitSource CloudInitSource) *VMBuilder { + var volume kubevirtv1.Volume + switch cloudInitSource.CloudInitType { + case CloudInitTypeNoCloud: + volume = kubevirtv1.Volume{ + Name: diskName, + VolumeSource: kubevirtv1.VolumeSource{ + CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ + UserData: cloudInitSource.UserData, + UserDataBase64: cloudInitSource.UserDataBase64, + NetworkData: cloudInitSource.NetworkData, + NetworkDataBase64: cloudInitSource.NetworkDataBase64, + }, + }, + } + if cloudInitSource.UserDataSecretName != "" { + volume.VolumeSource.CloudInitNoCloud.UserDataSecretRef = &corev1.LocalObjectReference{ + Name: cloudInitSource.UserDataSecretName, + } + } + if cloudInitSource.NetworkDataSecretName != "" { + volume.VolumeSource.CloudInitNoCloud.UserDataSecretRef = &corev1.LocalObjectReference{ + Name: cloudInitSource.NetworkDataSecretName, + } + } + case CloudInitTypeConfigDrive: + volume = kubevirtv1.Volume{ + Name: diskName, + VolumeSource: kubevirtv1.VolumeSource{ + CloudInitConfigDrive: &kubevirtv1.CloudInitConfigDriveSource{ + UserData: cloudInitSource.UserData, + UserDataBase64: cloudInitSource.UserDataBase64, + NetworkData: cloudInitSource.NetworkData, + NetworkDataBase64: cloudInitSource.NetworkDataBase64, + }, + }, + } + if cloudInitSource.UserDataSecretName != "" { + volume.VolumeSource.CloudInitConfigDrive.UserDataSecretRef = &corev1.LocalObjectReference{ + Name: cloudInitSource.UserDataSecretName, + } + } + if cloudInitSource.NetworkDataSecretName != "" { + volume.VolumeSource.CloudInitConfigDrive.UserDataSecretRef = &corev1.LocalObjectReference{ + Name: cloudInitSource.NetworkDataSecretName, + } + } + } + v.Volume(diskName, volume) + return v +} + +func (v *VMBuilder) CloudInitDisk(diskName, diskBus string, isCDRom bool, bootOrder int, cloudInitSource CloudInitSource) *VMBuilder { + return v.Disk(diskName, diskBus, isCDRom, bootOrder).CloudInit(diskName, cloudInitSource) +} + +func (v *VMBuilder) SSHKey(sshKeyName string) *VMBuilder { + v.SSHNames = append(v.SSHNames, sshKeyName) + return v +} diff --git a/pkg/builder/disk.go b/pkg/builder/disk.go new file mode 100644 index 0000000000..220d9dc407 --- /dev/null +++ b/pkg/builder/disk.go @@ -0,0 +1,208 @@ +package builder + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + kubevirtv1 "kubevirt.io/client-go/api/v1" + cdiv1alpha1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1" +) + +const ( + StorageClassNamePrefix = "longhorn" + + DiskTypeDisk = "disk" + DiskTypeCDRom = "cd-rom" + + DiskBusVirtio = "virtio" + DiskBusScsi = "scsi" + DiskBusSata = "sata" + + PersistentVolumeModeBlock = "Block" + PersistentVolumeModeFilesystem = "Filesystem" + + PersistentVolumeAccessModeReadWriteOnce = "ReadWriteOnce" + PersistentVolumeAccessModeReadOnlyMany = "ReadOnlyMany" + PersistentVolumeAccessModeReadWriteMany = "ReadWriteMany" + + DefaultDiskSize = "10Gi" + DefaultImagePullPolicy = "IfNotPresent" +) + +type DataVolumeOption struct { + ImageID string + DownloadURL string + VolumeMode corev1.PersistentVolumeMode + AccessMode corev1.PersistentVolumeAccessMode + StorageClassName *string +} + +func UintPtr(in int) *uint { + var out *uint + u := uint(in) + if in > 0 { + out = &u + } + return out +} + +func BuildImageStorageClassName(namespace, name string) string { + if namespace != "" { + return StorageClassNamePrefix + "-" + namespace + "-" + name + } + return StorageClassNamePrefix + "-" + name +} + +func (v *VMBuilder) Disk(diskName, diskBus string, isCDRom bool, bootOrder int) *VMBuilder { + var ( + exist bool + index int + disks = v.VirtualMachine.Spec.Template.Spec.Domain.Devices.Disks + ) + for i, disk := range disks { + if disk.Name == diskName { + exist = true + index = i + break + } + } + diskDevice := kubevirtv1.DiskDevice{ + Disk: &kubevirtv1.DiskTarget{ + Bus: diskBus, + }, + } + if isCDRom { + diskDevice = kubevirtv1.DiskDevice{ + CDRom: &kubevirtv1.CDRomTarget{ + Bus: diskBus, + }, + } + } + disk := kubevirtv1.Disk{ + Name: diskName, + BootOrder: UintPtr(bootOrder), + DiskDevice: diskDevice, + } + if exist { + disks[index] = disk + } else { + disks = append(disks, disk) + } + v.VirtualMachine.Spec.Template.Spec.Domain.Devices.Disks = disks + return v +} + +func (v *VMBuilder) Volume(diskName string, volume kubevirtv1.Volume) *VMBuilder { + var ( + exist bool + index int + volumes = v.VirtualMachine.Spec.Template.Spec.Volumes + ) + for i, e := range volumes { + if e.Name == diskName { + exist = true + index = i + break + } + } + + if exist { + volumes[index] = volume + } else { + volumes = append(volumes, volume) + } + v.VirtualMachine.Spec.Template.Spec.Volumes = volumes + return v +} + +func (v *VMBuilder) ExistingDataVolume(diskName, dataVolumeName string) *VMBuilder { + return v.Volume(diskName, kubevirtv1.Volume{ + Name: diskName, + VolumeSource: kubevirtv1.VolumeSource{ + DataVolume: &kubevirtv1.DataVolumeSource{ + Name: dataVolumeName, + }, + }, + }) +} + +func (v *VMBuilder) ExistingVolumeDisk(diskName, diskBus string, isCDRom bool, bootOrder int, dataVolumeName string) *VMBuilder { + return v.Disk(diskName, diskBus, isCDRom, bootOrder).ExistingDataVolume(diskName, dataVolumeName) +} + +func (v *VMBuilder) ContainerDiskVolume(diskName, imageName, ImagePullPolicy string) *VMBuilder { + return v.Volume(diskName, kubevirtv1.Volume{ + Name: diskName, + VolumeSource: kubevirtv1.VolumeSource{ + ContainerDisk: &kubevirtv1.ContainerDiskSource{ + Image: imageName, + ImagePullPolicy: corev1.PullPolicy(ImagePullPolicy), + }, + }, + }) +} + +func (v *VMBuilder) ContainerDisk(diskName, diskBus string, isCDRom bool, bootOrder int, imageName, ImagePullPolicy string) *VMBuilder { + return v.Disk(diskName, diskBus, isCDRom, bootOrder).ContainerDiskVolume(diskName, imageName, ImagePullPolicy) +} + +func (v *VMBuilder) DataVolume(diskName, diskSize, dataVolumeName string, opt *DataVolumeOption) *VMBuilder { + if opt == nil { + opt = &DataVolumeOption{ + VolumeMode: corev1.PersistentVolumeBlock, + AccessMode: corev1.ReadWriteMany, + } + } + if dataVolumeName == "" { + dataVolumeName = fmt.Sprintf("%s-%s-%s", v.VirtualMachine.Name, diskName, rand.String(5)) + } + // DataVolumeTemplates + dataVolumeTemplates := v.VirtualMachine.Spec.DataVolumeTemplates + dataVolumeSpecSource := cdiv1alpha1.DataVolumeSource{ + Blank: &cdiv1alpha1.DataVolumeBlankImage{}, + } + + if opt.DownloadURL != "" { + dataVolumeSpecSource = cdiv1alpha1.DataVolumeSource{ + HTTP: &cdiv1alpha1.DataVolumeSourceHTTP{ + URL: opt.DownloadURL, + }, + } + } + dataVolumeTemplate := kubevirtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataVolumeName, + }, + Spec: cdiv1alpha1.DataVolumeSpec{ + Source: dataVolumeSpecSource, + PVC: &corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + opt.AccessMode, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(diskSize), + }, + }, + VolumeMode: &opt.VolumeMode, + StorageClassName: opt.StorageClassName, + }, + }, + } + if opt.ImageID != "" { + dataVolumeTemplate.Annotations = map[string]string{ + AnnotationKeyImageID: opt.ImageID, + } + } + dataVolumeTemplates = append(dataVolumeTemplates, dataVolumeTemplate) + v.VirtualMachine.Spec.DataVolumeTemplates = dataVolumeTemplates + + return v.ExistingDataVolume(diskName, dataVolumeName) +} + +func (v *VMBuilder) DataVolumeDisk(diskName, diskBus string, isCDRom bool, bootOrder int, diskSize, dataVolumeName string, opt *DataVolumeOption) *VMBuilder { + return v.Disk(diskName, diskBus, isCDRom, bootOrder).DataVolume(diskName, diskSize, dataVolumeName, opt) +} diff --git a/pkg/builder/network.go b/pkg/builder/network.go new file mode 100644 index 0000000000..1c4fbe9787 --- /dev/null +++ b/pkg/builder/network.go @@ -0,0 +1,70 @@ +package builder + +import ( + kubevirtv1 "kubevirt.io/client-go/api/v1" +) + +const ( + NetworkInterfaceTypeBridge = "bridge" + NetworkInterfaceTypeMasquerade = "masquerade" + + LabelKeyNetworkType = "networks.harvesterhci.io/type" + + NetworkTypeVLAN = "L2VlanNetwork" + NetworkTypeCustom = "Custom" + + NetworkVLANConfigTemplate = `{"cniVersion":"0.3.1","name":"%s","type":"bridge","bridge":"harvester-br0","promiscMode":true,"vlan":%d,"ipam":{}}` +) + +func (v *VMBuilder) NetworkInterface(interfaceName, interfaceModel, interfaceMACAddress, interfaceType, networkName string) *VMBuilder { + v.Interface(interfaceName, interfaceModel, interfaceMACAddress, interfaceType) + v.Network(interfaceName, networkName) + return v +} + +func (v *VMBuilder) Network(interfaceName, networkName string) *VMBuilder { + networks := v.VirtualMachine.Spec.Template.Spec.Networks + network := kubevirtv1.Network{ + Name: interfaceName, + } + if networkName != "" { + network.NetworkSource = kubevirtv1.NetworkSource{ + Multus: &kubevirtv1.MultusNetwork{ + NetworkName: networkName, + Default: false, + }, + } + } else { + network.NetworkSource = kubevirtv1.NetworkSource{ + Pod: &kubevirtv1.PodNetwork{}, + } + } + networks = append(networks, network) + v.VirtualMachine.Spec.Template.Spec.Networks = networks + return v +} + +func (v *VMBuilder) Interface(interfaceName, interfaceModel, interfaceMACAddress string, interfaceType string) *VMBuilder { + interfaces := v.VirtualMachine.Spec.Template.Spec.Domain.Devices.Interfaces + networkInterface := kubevirtv1.Interface{ + Name: interfaceName, + Model: interfaceModel, + MacAddress: interfaceMACAddress, + InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ + Bridge: &kubevirtv1.InterfaceBridge{}, + }, + } + switch interfaceType { + case NetworkInterfaceTypeBridge: + networkInterface.InterfaceBindingMethod = kubevirtv1.InterfaceBindingMethod{ + Bridge: &kubevirtv1.InterfaceBridge{}, + } + default: + networkInterface.InterfaceBindingMethod = kubevirtv1.InterfaceBindingMethod{ + Masquerade: &kubevirtv1.InterfaceMasquerade{}, + } + } + interfaces = append(interfaces, networkInterface) + v.VirtualMachine.Spec.Template.Spec.Domain.Devices.Interfaces = interfaces + return v +} diff --git a/pkg/builder/vm.go b/pkg/builder/vm.go new file mode 100644 index 0000000000..1b0badca31 --- /dev/null +++ b/pkg/builder/vm.go @@ -0,0 +1,227 @@ +package builder + +import ( + "encoding/json" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + kubevirtv1 "kubevirt.io/client-go/api/v1" +) + +const ( + defaultVMGenerateName = "harv-" + defaultVMNamespace = "default" + + defaultVMCPUCores = 1 + defaultVMMemory = "256Mi" + + HarvesterAPIGroup = "harvesterhci.io" + LabelAnnotationPrefixHarvester = HarvesterAPIGroup + "/" + LabelKeyVirtualMachineCreator = LabelAnnotationPrefixHarvester + "creator" + LabelKeyVirtualMachineName = LabelAnnotationPrefixHarvester + "vmName" + AnnotationKeyVirtualMachineSSHNames = LabelAnnotationPrefixHarvester + "sshNames" + AnnotationKeyVirtualMachineDiskNames = LabelAnnotationPrefixHarvester + "diskNames" + AnnotationKeyImageID = LabelAnnotationPrefixHarvester + "imageId" + + AnnotationPrefixCattleField = "field.cattle.io/" + LabelPrefixHarvesterTag = "tag.harvesterhci.io/" + AnnotationKeyDescription = AnnotationPrefixCattleField + "description" +) + +type VMBuilder struct { + VirtualMachine *kubevirtv1.VirtualMachine + SSHNames []string + DataVolumeNames []string + InterfaceNames []string +} + +func NewVMBuilder(creator string) *VMBuilder { + vmLabels := map[string]string{ + LabelKeyVirtualMachineCreator: creator, + } + objectMeta := metav1.ObjectMeta{ + Namespace: defaultVMNamespace, + GenerateName: defaultVMGenerateName, + Labels: vmLabels, + Annotations: map[string]string{}, + } + running := pointer.BoolPtr(false) + cpu := &kubevirtv1.CPU{ + Cores: defaultVMCPUCores, + } + resources := kubevirtv1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(defaultVMMemory), + }, + } + template := &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: vmLabels, + }, + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + CPU: cpu, + Devices: kubevirtv1.Devices{ + Disks: []kubevirtv1.Disk{}, + Interfaces: []kubevirtv1.Interface{}, + }, + Resources: resources, + }, + Affinity: &corev1.Affinity{}, + Networks: []kubevirtv1.Network{}, + Volumes: []kubevirtv1.Volume{}, + }, + } + + vm := &kubevirtv1.VirtualMachine{ + ObjectMeta: objectMeta, + Spec: kubevirtv1.VirtualMachineSpec{ + Running: running, + Template: template, + DataVolumeTemplates: []kubevirtv1.DataVolumeTemplateSpec{}, + }, + } + return &VMBuilder{ + VirtualMachine: vm, + } +} + +func (v *VMBuilder) Name(name string) *VMBuilder { + v.VirtualMachine.ObjectMeta.Name = name + v.VirtualMachine.ObjectMeta.GenerateName = "" + v.VirtualMachine.Spec.Template.ObjectMeta.Labels[LabelKeyVirtualMachineName] = name + return v +} + +func (v *VMBuilder) Namespace(namespace string) *VMBuilder { + v.VirtualMachine.ObjectMeta.Namespace = namespace + return v +} + +func (v *VMBuilder) MachineType(machineType string) *VMBuilder { + v.VirtualMachine.Spec.Template.Spec.Domain.Machine.Type = machineType + return v +} + +func (v *VMBuilder) HostName(hostname string) *VMBuilder { + v.VirtualMachine.Spec.Template.Spec.Hostname = hostname + return v +} + +func (v *VMBuilder) Description(description string) *VMBuilder { + if v.VirtualMachine.ObjectMeta.Annotations == nil { + v.VirtualMachine.ObjectMeta.Annotations = map[string]string{} + } + v.VirtualMachine.ObjectMeta.Annotations[AnnotationKeyDescription] = description + return v +} + +func (v *VMBuilder) Labels(labels map[string]string) *VMBuilder { + if v.VirtualMachine.ObjectMeta.Labels == nil { + v.VirtualMachine.ObjectMeta.Labels = labels + } + for key, value := range labels { + v.VirtualMachine.ObjectMeta.Labels[key] = value + } + return v +} + +func (v *VMBuilder) Annotations(annotations map[string]string) *VMBuilder { + if v.VirtualMachine.ObjectMeta.Annotations == nil { + v.VirtualMachine.ObjectMeta.Annotations = annotations + } + for key, value := range annotations { + v.VirtualMachine.ObjectMeta.Annotations[key] = value + } + return v +} + +func (v *VMBuilder) Memory(memory string) *VMBuilder { + v.VirtualMachine.Spec.Template.Spec.Domain.Resources.Requests = corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(memory), + } + return v +} + +func (v *VMBuilder) CPU(cores int) *VMBuilder { + v.VirtualMachine.Spec.Template.Spec.Domain.CPU.Cores = uint32(cores) + return v +} + +func (v *VMBuilder) EvictionStrategy(liveMigrate bool) *VMBuilder { + if liveMigrate { + evictionStrategy := kubevirtv1.EvictionStrategyLiveMigrate + v.VirtualMachine.Spec.Template.Spec.EvictionStrategy = &evictionStrategy + } + return v +} + +func (v *VMBuilder) DefaultPodAntiAffinity() *VMBuilder { + podAffinityTerm := corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: LabelKeyVirtualMachineCreator, + Operator: metav1.LabelSelectorOpExists, + }, + }, + }, + TopologyKey: corev1.LabelHostname, + } + return v.PodAntiAffinity(podAffinityTerm, true, 100) +} + +func (v *VMBuilder) PodAntiAffinity(podAffinityTerm corev1.PodAffinityTerm, soft bool, weight int32) *VMBuilder { + podAffinity := &corev1.PodAntiAffinity{} + if soft { + podAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.WeightedPodAffinityTerm{ + { + Weight: weight, + PodAffinityTerm: podAffinityTerm, + }, + } + } else { + podAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []corev1.PodAffinityTerm{ + podAffinityTerm, + } + } + v.VirtualMachine.Spec.Template.Spec.Affinity.PodAntiAffinity = podAffinity + return v +} + +func (v *VMBuilder) Run(start bool) *VMBuilder { + v.VirtualMachine.Spec.Running = pointer.BoolPtr(start) + return v +} + +func (v *VMBuilder) VM() (*kubevirtv1.VirtualMachine, error) { + if v.VirtualMachine.Spec.Template.ObjectMeta.Annotations == nil { + v.VirtualMachine.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + sshNames, err := json.Marshal(v.SSHNames) + if err != nil { + return nil, err + } + v.VirtualMachine.Spec.Template.ObjectMeta.Annotations[AnnotationKeyVirtualMachineSSHNames] = string(sshNames) + + volumes := v.VirtualMachine.Spec.Template.Spec.Volumes + for _, volume := range volumes { + if volume.DataVolume != nil { + v.DataVolumeNames = append(v.DataVolumeNames, volume.DataVolume.Name) + } + } + dataVolumeNames, err := json.Marshal(v.DataVolumeNames) + if err != nil { + return nil, err + } + v.VirtualMachine.Spec.Template.ObjectMeta.Annotations[AnnotationKeyVirtualMachineDiskNames] = string(dataVolumeNames) + + return v.VirtualMachine, nil +} + +func (v *VMBuilder) Update(vm *kubevirtv1.VirtualMachine) *VMBuilder { + v.VirtualMachine = vm + return v +} diff --git a/tests/integration/api/network_apis_test.go b/tests/integration/api/network_apis_test.go index 4ddf465269..ecbe243241 100644 --- a/tests/integration/api/network_apis_test.go +++ b/tests/integration/api/network_apis_test.go @@ -9,6 +9,7 @@ import ( "github.com/tidwall/gjson" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/harvester/harvester/pkg/builder" "github.com/harvester/harvester/pkg/config" ctlkubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" . "github.com/harvester/harvester/tests/framework/dsl" @@ -17,12 +18,8 @@ import ( ) const ( - testNetworkNamespace = "default" - testNetworkTypeVlan = "L2VlanNetwork" - testBridgeVID = 100 - testNetworkBridgeTemplate = ` -{"cniVersion":"0.3.1","name":"%s","type":"bridge","bridge":"harvester-br0","promiscMode":true,"vlan":%d,"ipam":{}} -` + testNetworkNamespace = "default" + testBridgeVID = 100 ) type BridgeNetwork struct { @@ -31,19 +28,19 @@ type BridgeNetwork struct { func NewBridgeNetwork(name string, vid int) *BridgeNetwork { bridgeNetwork := BridgeNetwork{ - NAD: NewNAD(name, testNetworkTypeVlan, NewBridgeNetworkConfig(name, vid)), + NAD: NewNAD(name, builder.NetworkTypeVLAN, NewBridgeNetworkConfig(name, vid)), } return &bridgeNetwork } func NewBridgeNetworkConfig(name string, vid int) string { - return fmt.Sprintf(testNetworkBridgeTemplate, name, vid) + return fmt.Sprintf(builder.NetworkVLANConfigTemplate, name, vid) } func NewNAD(name, networkType, config string) *cniv1.NetworkAttachmentDefinition { networkLabels := map[string]string{ - "test.harvesterhci.io": "harvester-test", - "networks.harvesterhci.io/type": networkType, + "test.harvesterhci.io": "harvester-test", + builder.LabelKeyNetworkType: networkType, } return &cniv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{ @@ -61,7 +58,7 @@ var _ = Describe("verify network APIs", func() { var ( scaled *config.Scaled - vmBuilder *VMBuilder + vmBuilder *builder.VMBuilder vmNamespace string vmController ctlkubevirtv1.VirtualMachineController networkName string @@ -151,7 +148,11 @@ var _ = Describe("verify network APIs", func() { Specify("verify the delete action", func() { By("create a vm use this network") - vm, err := vmController.Create(vmBuilder.Container().Network(networkName).VM()) + toCreate, err := vmBuilder.ContainerDisk(testVMContainerDiskName, testVMDefaultDiskBus, false, 1, testVMContainerDiskImageName, testVMContainerDiskImagePullPolicy). + NetworkInterface(testVMInterfaceName, testVMInterfaceModel, "", builder.NetworkInterfaceTypeBridge, networkName). + VM() + MustNotError(err) + vm, err := vmController.Create(toCreate) MustNotError(err) vmName := vm.Name MustVMExist(vmController, vmNamespace, vmName) diff --git a/tests/integration/api/vm_apis_test.go b/tests/integration/api/vm_apis_test.go index bc61336b09..58500937fe 100644 --- a/tests/integration/api/vm_apis_test.go +++ b/tests/integration/api/vm_apis_test.go @@ -12,6 +12,7 @@ import ( kubevirtv1 "kubevirt.io/client-go/api/v1" apivm "github.com/harvester/harvester/pkg/api/vm" + "github.com/harvester/harvester/pkg/builder" "github.com/harvester/harvester/pkg/config" ctlkubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" . "github.com/harvester/harvester/tests/framework/dsl" @@ -63,7 +64,11 @@ var _ = Describe("verify vm APIs", func() { // create By("create a virtual machine should fail if name missing") - vm := NewDefaultTestVMBuilder(testResourceLabels).Name("").Blank().VM() + vm, err := NewDefaultTestVMBuilder(testResourceLabels).Name(""). + NetworkInterface(testVMInterfaceName, testVMInterfaceModel, "", builder.NetworkInterfaceTypeMasquerade, ""). + DataVolumeDisk(testVMBlankDiskName, testVMDefaultDiskBus, false, 1, testVMDiskSize, "", nil). + VM() + MustNotError(err) respCode, respBody, err := helper.PostObject(vmsAPI, vm) MustRespCodeIs(http.StatusUnprocessableEntity, "create vm", err, respCode, respBody) @@ -76,11 +81,17 @@ var _ = Describe("verify vm APIs", func() { Address: "10.5.2.100/24", Gateway: "10.5.2.1", } - vm = NewDefaultTestVMBuilder(testResourceLabels).Name(vmName). - Container(). - Blank(). - CloudInit(vmCloudInit). - Run() + userData := fmt.Sprintf(testVMCloudInitUserDataTemplate, vmCloudInit.UserName, vmCloudInit.Password) + networkData := fmt.Sprintf(testVMCloudInitNetworkDataTemplate, vmCloudInit.Address, vmCloudInit.Gateway) + vm, err = NewDefaultTestVMBuilder(testResourceLabels).Name(vmName). + NetworkInterface(testVMInterfaceName, testVMInterfaceModel, "", builder.NetworkInterfaceTypeMasquerade, ""). + ContainerDisk(testVMContainerDiskName, testVMDefaultDiskBus, false, 1, testVMContainerDiskImageName, testVMContainerDiskImagePullPolicy). + CloudInitDisk(testVMCloudInitDiskName, testVMDefaultDiskBus, false, 0, builder.CloudInitSource{ + CloudInitType: builder.CloudInitTypeNoCloud, + UserData: userData, + NetworkData: networkData, + }).Run(true).VM() + MustNotError(err) respCode, respBody, err = helper.PostObject(vmsAPI, vm) MustRespCodeIs(http.StatusCreated, "create vm", err, respCode, respBody) @@ -97,22 +108,21 @@ var _ = Describe("verify vm APIs", func() { // edit By("when edit virtual machine") - updatedCPUCore := uint32(2) - updatedMemory := "200Mi" - vm = NewVMBuilder(vm). - CPU(updatedCPUCore). - Memory(updatedMemory). - CDRom(). - VM() + vm, err = builder.NewVMBuilder(testCreator).Update(vm).CPU(testVMUpdatedCPUCore).Memory(testVMUpdatedMemory). + DataVolumeDisk(testVMCDRomDiskName, testVMCDRomBus, true, 2, testVMDiskSize, "", &builder.DataVolumeOption{ + VolumeMode: builder.PersistentVolumeModeFilesystem, + AccessMode: builder.PersistentVolumeAccessModeReadWriteOnce, + }).VM() + MustNotError(err) respCode, respBody, err = helper.PutObject(vmURL, vm) MustRespCodeIs(http.StatusOK, "put edit action", err, respCode, respBody) By("then the virtual machine is changed") AfterVMRunning(vmController, vmNamespace, vmName, func(vm *kubevirtv1.VirtualMachine) bool { spec := vm.Spec.Template.Spec - MustEqual(len(spec.Domain.Devices.Disks), 4) - MustEqual(spec.Domain.CPU.Cores, updatedCPUCore) - MustEqual(spec.Domain.Resources.Requests[corev1.ResourceMemory], resource.MustParse(updatedMemory)) + MustEqual(len(spec.Domain.Devices.Disks), 3) + MustEqual(spec.Domain.CPU.Cores, uint32(testVMUpdatedCPUCore)) + MustEqual(spec.Domain.Resources.Requests[corev1.ResourceMemory], resource.MustParse(testVMUpdatedMemory)) return true }) @@ -131,10 +141,10 @@ var _ = Describe("verify vm APIs", func() { By("then the virtual machine instance is changed") AfterVMIRestarted(vmController, vmNamespace, vmName, vmiController, vmiUID, func(vmi *kubevirtv1.VirtualMachineInstance) bool { - spec := vm.Spec.Template.Spec - MustEqual(len(spec.Domain.Devices.Disks), 4) - MustEqual(spec.Domain.CPU.Cores, updatedCPUCore) - MustEqual(spec.Domain.Resources.Requests[corev1.ResourceMemory], resource.MustParse(updatedMemory)) + spec := vmi.Spec + MustEqual(len(spec.Domain.Devices.Disks), 3) + MustEqual(spec.Domain.CPU.Cores, uint32(testVMUpdatedCPUCore)) + MustEqual(spec.Domain.Resources.Requests[corev1.ResourceMemory], resource.MustParse(testVMUpdatedMemory)) return true }) diff --git a/tests/integration/api/vm_apis_utils_test.go b/tests/integration/api/vm_apis_utils_test.go index 6283ba9bb1..d06236d251 100644 --- a/tests/integration/api/vm_apis_utils_test.go +++ b/tests/integration/api/vm_apis_utils_test.go @@ -8,36 +8,34 @@ import ( "strings" "github.com/onsi/ginkgo" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" - kubevirtv1 "kubevirt.io/client-go/api/v1" - cdiv1alpha1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1" - "github.com/harvester/harvester/tests/framework/env" - "github.com/harvester/harvester/tests/framework/fuzz" + "github.com/harvester/harvester/pkg/builder" ) const ( + testCreator = "harvester-integration-test" testVMGenerateName = "test-" testVMNamespace = "default" - testVMCPUCores = 1 - testVMMemory = "256Mi" + testVMCPUCores = 1 + testVMMemory = "256Mi" + testVMUpdatedCPUCore = 2 + testVMUpdatedMemory = "200Mi" - testVMManagementNetworkName = "default" - testVMManagementInterfaceName = "default" - testVMInterfaceModel = "virtio" + testVMDiskSize = "10Mi" + testVMInterfaceName = "default" + testVMInterfaceModel = "virtio" + + testVMBlankDiskName = "blankdisk" testVMCDRomDiskName = "cdromdisk" - testVMCloudInitDiskName = "cloudinitdisk" testVMContainerDiskName = "containerdisk" testVMContainerDiskImageName = "kubevirt/fedora-cloud-container-disk-demo:v0.35.0" - testVMContainerDiskImagePullPolicy = corev1.PullIfNotPresent + testVMContainerDiskImagePullPolicy = builder.DefaultImagePullPolicy + testVMCloudInitDiskName = builder.CloudInitDiskName - testVMDefaultDiskBus = "virtio" - testVMCDRomBus = "sata" + testVMDefaultDiskBus = builder.DiskBusVirtio + testVMCDRomBus = builder.DiskBusSata testVMCloudInitUserDataTemplate = ` #cloud-config @@ -142,324 +140,7 @@ func CreateTmpFile(dir, pattern, content string, mode os.FileMode) (string, erro return fileName, err } -type VMBuilder struct { - vm *kubevirtv1.VirtualMachine -} - -func NewVMBuilder(vm *kubevirtv1.VirtualMachine) *VMBuilder { - return &VMBuilder{ - vm: vm, - } -} - -func NewDefaultTestVMBuilder(labels map[string]string) *VMBuilder { - objectMeta := metav1.ObjectMeta{ - Namespace: testVMNamespace, - GenerateName: testVMGenerateName, - Labels: labels, - } - running := pointer.BoolPtr(false) - cpu := &kubevirtv1.CPU{ - Cores: testVMCPUCores, - } - resources := kubevirtv1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse(testVMMemory), - }, - } - interfaces := []kubevirtv1.Interface{ - { - Name: testVMManagementInterfaceName, - Model: testVMInterfaceModel, - InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ - Masquerade: &kubevirtv1.InterfaceMasquerade{}, - }, - }, - } - networks := []kubevirtv1.Network{ - { - Name: testVMManagementNetworkName, - NetworkSource: kubevirtv1.NetworkSource{ - Pod: &kubevirtv1.PodNetwork{}, - }, - }, - } - template := &kubevirtv1.VirtualMachineInstanceTemplateSpec{ - Spec: kubevirtv1.VirtualMachineInstanceSpec{ - Domain: kubevirtv1.DomainSpec{ - CPU: cpu, - Devices: kubevirtv1.Devices{ - Disks: []kubevirtv1.Disk{}, - Interfaces: interfaces, - }, - Resources: resources, - }, - Networks: networks, - Volumes: []kubevirtv1.Volume{}, - }, - } - vm := &kubevirtv1.VirtualMachine{ - ObjectMeta: objectMeta, - Spec: kubevirtv1.VirtualMachineSpec{ - Running: running, - Template: template, - DataVolumeTemplates: []kubevirtv1.DataVolumeTemplateSpec{}, - }, - } - return &VMBuilder{ - vm: vm, - } -} - -func (v *VMBuilder) Name(name string) *VMBuilder { - v.vm.ObjectMeta.Name = name - v.vm.ObjectMeta.GenerateName = "" - return v -} - -func (v *VMBuilder) Namespace(namespace string) *VMBuilder { - v.vm.ObjectMeta.Namespace = namespace - return v -} - -func (v *VMBuilder) Memory(memory string) *VMBuilder { - v.vm.Spec.Template.Spec.Domain.Resources.Requests = corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse(memory), - } - return v -} - -func (v *VMBuilder) CPU(cores uint32) *VMBuilder { - v.vm.Spec.Template.Spec.Domain.CPU.Cores = cores - return v -} - -func (v *VMBuilder) Blank() *VMBuilder { - v.DataVolume("disk-blank", "10Mi", "") - return v -} - -func (v *VMBuilder) Image(imageName string) *VMBuilder { - return v.DataVolume("disk-image", "10Gi", fmt.Sprintf("longhorn-%s", imageName)) -} - -func (v *VMBuilder) DataVolume(diskName, storageSize string, storageClass string) *VMBuilder { - volumeMode := corev1.PersistentVolumeFilesystem - if env.IsE2ETestsEnabled() { - volumeMode = corev1.PersistentVolumeBlock - } - dataVolumeName := fmt.Sprintf("%s-%s-%s", v.vm.Name, diskName, fuzz.String(5)) - // DataVolumeTemplates - dataVolumeTemplates := v.vm.Spec.DataVolumeTemplates - dataVolumeSpecSource := cdiv1alpha1.DataVolumeSource{ - Blank: &cdiv1alpha1.DataVolumeBlankImage{}, - } - - dataVolumeTemplate := kubevirtv1.DataVolumeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: dataVolumeName, - Labels: nil, - Annotations: nil, - }, - Spec: cdiv1alpha1.DataVolumeSpec{ - Source: dataVolumeSpecSource, - PVC: &corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(storageSize), - }, - }, - VolumeMode: &volumeMode, - }, - }, - } - - if storageClass != "" { - dataVolumeTemplate.Spec.PVC.StorageClassName = &storageClass - } else if env.IsE2ETestsEnabled() { - dataVolumeTemplate.Spec.PVC.StorageClassName = pointer.StringPtr("longhorn") - } - - dataVolumeTemplates = append(dataVolumeTemplates, dataVolumeTemplate) - v.vm.Spec.DataVolumeTemplates = dataVolumeTemplates - // Disks - disks := v.vm.Spec.Template.Spec.Domain.Devices.Disks - disks = append(disks, kubevirtv1.Disk{ - Name: diskName, - DiskDevice: kubevirtv1.DiskDevice{ - Disk: &kubevirtv1.DiskTarget{ - Bus: testVMDefaultDiskBus, - }, - }, - }) - v.vm.Spec.Template.Spec.Domain.Devices.Disks = disks - // Volumes - volumes := v.vm.Spec.Template.Spec.Volumes - volumes = append(volumes, kubevirtv1.Volume{ - Name: diskName, - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ - Name: dataVolumeName, - }, - }, - }) - v.vm.Spec.Template.Spec.Volumes = volumes - return v -} - -func (v *VMBuilder) ExistingDataVolume(diskName, dataVolumeName string) *VMBuilder { - // Disks - disks := v.vm.Spec.Template.Spec.Domain.Devices.Disks - disks = append(disks, kubevirtv1.Disk{ - Name: diskName, - DiskDevice: kubevirtv1.DiskDevice{ - Disk: &kubevirtv1.DiskTarget{ - Bus: testVMDefaultDiskBus, - }, - }, - }) - v.vm.Spec.Template.Spec.Domain.Devices.Disks = disks - // Volumes - volumes := v.vm.Spec.Template.Spec.Volumes - volumes = append(volumes, kubevirtv1.Volume{ - Name: diskName, - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ - Name: dataVolumeName, - }, - }, - }) - v.vm.Spec.Template.Spec.Volumes = volumes - return v -} - -func (v *VMBuilder) ContainerDisk(diskName, imageName string, isCDRom bool) *VMBuilder { - // Disks - disks := v.vm.Spec.Template.Spec.Domain.Devices.Disks - diskDevice := kubevirtv1.DiskDevice{ - Disk: &kubevirtv1.DiskTarget{ - Bus: testVMDefaultDiskBus, - }, - } - if isCDRom { - diskDevice = kubevirtv1.DiskDevice{ - CDRom: &kubevirtv1.CDRomTarget{ - Bus: testVMCDRomBus, - }, - } - } - disks = append(disks, kubevirtv1.Disk{ - Name: diskName, - DiskDevice: diskDevice, - }) - v.vm.Spec.Template.Spec.Domain.Devices.Disks = disks - // Volumes - volumes := v.vm.Spec.Template.Spec.Volumes - volumes = append(volumes, kubevirtv1.Volume{ - Name: diskName, - VolumeSource: kubevirtv1.VolumeSource{ - ContainerDisk: &kubevirtv1.ContainerDiskSource{ - Image: imageName, - ImagePullPolicy: testVMContainerDiskImagePullPolicy, - }, - }, - }) - v.vm.Spec.Template.Spec.Volumes = volumes - return v -} - -func (v *VMBuilder) Container(containerImageName ...string) *VMBuilder { - imageName := testVMContainerDiskImageName - if len(containerImageName) > 0 { - imageName = containerImageName[0] - } - return v.ContainerDisk(testVMContainerDiskName, imageName, false) -} - -func (v *VMBuilder) CDRom(containerImageName ...string) *VMBuilder { - imageName := testVMContainerDiskImageName - if len(containerImageName) > 0 { - imageName = containerImageName[0] - } - return v.ContainerDisk(testVMCDRomDiskName, imageName, true) -} - -func (v *VMBuilder) CloudInit(vmCloudInit *VMCloudInit) *VMBuilder { - // Disks - disks := v.vm.Spec.Template.Spec.Domain.Devices.Disks - for _, disk := range disks { - if disk.Name == testVMCloudInitDiskName { - return v - } - } - - disks = append(disks, kubevirtv1.Disk{ - Name: testVMCloudInitDiskName, - DiskDevice: kubevirtv1.DiskDevice{ - Disk: &kubevirtv1.DiskTarget{ - Bus: testVMDefaultDiskBus, - }, - }, - }) - v.vm.Spec.Template.Spec.Domain.Devices.Disks = disks - // Volumes - var userData, networkData string - if vmCloudInit != nil { - if vmCloudInit.Password != "" { - userData = fmt.Sprintf(testVMCloudInitUserDataTemplate, vmCloudInit.UserName, vmCloudInit.Password) - } - if vmCloudInit.Address != "" && vmCloudInit.Gateway != "" { - networkData = fmt.Sprintf(testVMCloudInitNetworkDataTemplate, vmCloudInit.Address, vmCloudInit.Gateway) - } - } - volumes := v.vm.Spec.Template.Spec.Volumes - volumes = append(volumes, kubevirtv1.Volume{ - Name: testVMCloudInitDiskName, - VolumeSource: kubevirtv1.VolumeSource{ - CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ - UserData: userData, - NetworkData: networkData, - }, - }, - }) - v.vm.Spec.Template.Spec.Volumes = volumes - return v -} - -func (v *VMBuilder) Network(networkName string) *VMBuilder { - // Networks - networks := v.vm.Spec.Template.Spec.Networks - networks = append(networks, kubevirtv1.Network{ - Name: networkName, - NetworkSource: kubevirtv1.NetworkSource{ - Multus: &kubevirtv1.MultusNetwork{ - NetworkName: networkName, - Default: false, - }, - }, - }) - v.vm.Spec.Template.Spec.Networks = networks - // Interfaces - interfaces := v.vm.Spec.Template.Spec.Domain.Devices.Interfaces - interfaces = append(interfaces, kubevirtv1.Interface{ - Name: networkName, - Model: testVMInterfaceModel, - InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ - Bridge: &kubevirtv1.InterfaceBridge{}, - }, - }) - v.vm.Spec.Template.Spec.Domain.Devices.Interfaces = interfaces - return v -} - -func (v *VMBuilder) Run() *kubevirtv1.VirtualMachine { - v.vm.Spec.Running = pointer.BoolPtr(true) - return v.VM() -} - -func (v *VMBuilder) VM() *kubevirtv1.VirtualMachine { - return v.vm +func NewDefaultTestVMBuilder(labels map[string]string) *builder.VMBuilder { + return builder.NewVMBuilder(testCreator).Namespace(testVMNamespace).Labels(labels). + CPU(testVMCPUCores).Memory(testVMMemory).Run(false) } diff --git a/tests/integration/api/vm_backup_restore_test.go b/tests/integration/api/vm_backup_restore_test.go index 6b696f0877..819fd9071e 100644 --- a/tests/integration/api/vm_backup_restore_test.go +++ b/tests/integration/api/vm_backup_restore_test.go @@ -14,6 +14,7 @@ import ( apivm "github.com/harvester/harvester/pkg/api/vm" harvesterv1 "github.com/harvester/harvester/pkg/apis/harvesterhci.io/v1beta1" + "github.com/harvester/harvester/pkg/builder" "github.com/harvester/harvester/pkg/config" ctlharvesterv1 "github.com/harvester/harvester/pkg/generated/controllers/harvesterhci.io/v1beta1" ctlkubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" @@ -130,9 +131,16 @@ var _ = Describe("verify vm backup & restore APIs", func() { By("when create a VM using data volume") vmName := testVMGenerateName + fuzz.String(5) - vm := NewDefaultTestVMBuilder(testVMBackupLabels).Name(vmName). - DataVolume("root-disk", "2Gi", sourceImage). - Run() + dataVolumeOption := &builder.DataVolumeOption{ + VolumeMode: builder.PersistentVolumeModeBlock, + AccessMode: builder.PersistentVolumeAccessModeReadWriteMany, + DownloadURL: sourceImage, + } + vm, err := NewDefaultTestVMBuilder(testVMBackupLabels).Name(vmName). + NetworkInterface(testVMInterfaceName, testVMInterfaceModel, "", builder.NetworkInterfaceTypeMasquerade, ""). + DataVolumeDisk("root-disk", testVMDefaultDiskBus, false, 1, "2Gi", "", dataVolumeOption). + Run(true).VM() + MustNotError(err) respCode, respBody, err := helper.PostObject(vmsAPI, vm) MustRespCodeIs(http.StatusCreated, "create vm", err, respCode, respBody) diff --git a/tests/integration/api/vm_template_apis_test.go b/tests/integration/api/vm_template_apis_test.go index 30587f52a8..73095c14a3 100644 --- a/tests/integration/api/vm_template_apis_test.go +++ b/tests/integration/api/vm_template_apis_test.go @@ -140,7 +140,9 @@ var _ = Describe("verify vm template APIs", func() { By("create a vm template version", func() { templateVersion.Spec.TemplateID = templateID - templateVersion.Spec.VM = NewDefaultTestVMBuilder(testResourceLabels).vm.Spec + vm, err := NewDefaultTestVMBuilder(testResourceLabels).VM() + MustNotError(err) + templateVersion.Spec.VM = vm.Spec respCode, respBody, err := helper.PostObjectByYAML(templateVersionAPI, templateVersion) MustRespCodeIs(http.StatusCreated, "create template version", err, respCode, respBody)