Skip to content

Commit

Permalink
vSphere: support for VM Groups (#1847)
Browse files Browse the repository at this point in the history
* vSphere: support for VM Groups

Signed-off-by: Waleed Malik <ahmedwaleedmalik@gmail.com>

* Use break instead of continue to exit loop

Signed-off-by: Waleed Malik <ahmedwaleedmalik@gmail.com>

---------

Signed-off-by: Waleed Malik <ahmedwaleedmalik@gmail.com>
  • Loading branch information
ahmedwaleedmalik authored Aug 26, 2024
1 parent 107e351 commit 3ead298
Show file tree
Hide file tree
Showing 8 changed files with 167 additions and 21 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,5 @@ examples/*.srl
/vendor
.vscode
.gitpod.yml
cmd/machine-controller/__debug_bin
cmd/machine-controller/__debug_bin*
!pkg
3 changes: 2 additions & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ issues:
- func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec
- func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec
- func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec
- 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high'
- 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Create` is high'
- 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Validate` is high'
- "SA1019: s.server.IPv6 is deprecated"
exclude-dirs:
- pkg/machines
2 changes: 2 additions & 0 deletions examples/vsphere-machinedeployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ spec:
cluster: cl-1
# Automatically create anti affinity rules for machines
vmAntiAffinity: true
# Optional. Sets the VM group for the Machines in the MachineDeployment.
# vmGroup: "vmgroup-name"
cpus: 2
memoryMB: 2048
# Optional: Resize the root disk to this size. Must be bigger than the existing size
Expand Down
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/tinkerbell/tink v0.10.0
github.com/vmware/go-vcloud-director/v2 v2.25.0
github.com/vmware/govmomi v0.38.0
github.com/vmware/govmomi v0.42.0
github.com/vultr/govultr/v3 v3.9.0
go.anx.io/go-anxcloud v0.7.2
go.uber.org/zap v1.27.0
Expand Down Expand Up @@ -154,10 +154,10 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.23.0 // indirect
google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 // indirect
Expand Down
12 changes: 6 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,8 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7
github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4=
github.com/vmware/go-vcloud-director/v2 v2.25.0 h1:RcJ5FQRku3FvQktTi8YOZsRfvhfLm315Cme50M9x9MQ=
github.com/vmware/go-vcloud-director/v2 v2.25.0/go.mod h1:7Of1qJja+LLNKVegjZG7uuhhy6xgGg3q7Fkw2CEP+Tw=
github.com/vmware/govmomi v0.38.0 h1:UvQpLAOjDpO0JUxoPCXnEzOlEa/9kejO6K58qOFr6cM=
github.com/vmware/govmomi v0.38.0/go.mod h1:mtGWtM+YhTADHlCgJBiskSRPOZRsN9MSjPzaZLte/oQ=
github.com/vmware/govmomi v0.42.0 h1:MbvAlVfjNBE1mHMaQ7yOSop1KLB0/93x6VAGuCtjqtI=
github.com/vmware/govmomi v0.42.0/go.mod h1:1H5LWwsBif8HKZqbFp0FdoKTHyJE4FzL6ACequMKYQg=
github.com/vultr/govultr/v3 v3.9.0 h1:63V/22mpfquRA5DenJ9EF0VozHg0k+X4dhUWcDXHPyc=
github.com/vultr/govultr/v3 v3.9.0/go.mod h1:Rd8ebpXm7jxH3MDmhnEs+zrlYW212ouhx+HeUMfHm2o=
github.com/wI2L/jsondiff v0.2.0 h1:dE00WemBa1uCjrzQUUTE/17I6m5qAaN0EMFOg2Ynr/k=
Expand Down Expand Up @@ -580,8 +580,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -634,8 +634,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand Down
23 changes: 19 additions & 4 deletions pkg/cloudprovider/provider/vsphere/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type Config struct {
MemoryMB int64
DiskSizeGB *int64
Tags []tags.Tag
VMGroup string
}

// Ensures that Server implements Instance interface.
Expand Down Expand Up @@ -210,6 +211,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p
return nil, nil, nil, err
}

c.VMGroup, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VMGroup)
if err != nil {
return nil, nil, nil, err
}

c.CPUs = rawConfig.CPUs
c.MemoryMB = rawConfig.MemoryMB
c.DiskSizeGB = rawConfig.DiskSizeGB
Expand Down Expand Up @@ -311,10 +317,13 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec cl
}
}

if config.VMAntiAffinity {
if config.Cluster == "" {
return fmt.Errorf("cluster is required for vm anti affinity")
}
if config.VMAntiAffinity && config.Cluster == "" {
return fmt.Errorf("cluster is required for vm anti affinity")
} else if config.VMGroup != "" && config.Cluster == "" {
return fmt.Errorf("cluster is required for vm group")
}

if config.Cluster != "" {
_, err = session.Finder.ClusterComputeResource(ctx, config.Cluster)
if err != nil {
return fmt.Errorf("failed to get cluster %q, %w", config.Cluster, err)
Expand Down Expand Up @@ -376,6 +385,12 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine *
return nil, fmt.Errorf("failed to attach tags: %w", err)
}

if config.VMGroup != "" {
if err := p.addToVMGroup(ctx, log, session, machine, config); err != nil {
return nil, fmt.Errorf("failed to add VM to VM group: %w", err)
}
}

if config.VMAntiAffinity {
if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil {
return nil, fmt.Errorf("failed to add VM to anti affinity rule: %w", err)
Expand Down
15 changes: 9 additions & 6 deletions pkg/cloudprovider/provider/vsphere/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,15 @@ type RawConfig struct {
DatastoreCluster providerconfigtypes.ConfigVarString `json:"datastoreCluster"`
Datastore providerconfigtypes.ConfigVarString `json:"datastore"`

CPUs int32 `json:"cpus"`
MemoryMB int64 `json:"memoryMB"`
DiskSizeGB *int64 `json:"diskSizeGB,omitempty"`
Tags []Tag `json:"tags,omitempty"`
AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"`
VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"`
CPUs int32 `json:"cpus"`
MemoryMB int64 `json:"memoryMB"`
DiskSizeGB *int64 `json:"diskSizeGB,omitempty"`
Tags []Tag `json:"tags,omitempty"`
AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"`

// Placement rules
VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"`
VMGroup providerconfigtypes.ConfigVarString `json:"vmGroup,omitempty"`
}

// Tag represents vsphere tag.
Expand Down
125 changes: 125 additions & 0 deletions pkg/cloudprovider/provider/vsphere/vmgroup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
Copyright 2024 The Machine Controller Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsphere

import (
"context"
"fmt"
"strings"

clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1"

"github.com/vmware/govmomi/vim25/types"
"go.uber.org/zap"
)

func (p *provider) addToVMGroup(ctx context.Context, log *zap.SugaredLogger, session *Session, machine *clusterv1alpha1.Machine, config *Config) error {
lock.Lock()
defer lock.Unlock()

// Check if the VM group exists
vmGroup, err := findVMGroup(ctx, session, config.Cluster, config.VMGroup)
if err != nil {
return err
}

// We have to find all VMs in the folder and add them to the VM group. VMGroup only contains VM reference ID which is not enough to
// identify the VM by name.
machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")]
vmsInFolder, err := session.Finder.VirtualMachineList(ctx, strings.Join([]string{config.Folder, "*"}, "/"))
if err != nil {
return fmt.Errorf("failed to find VMs in folder: %w", err)
}

var vmRefs []types.ManagedObjectReference
for _, vm := range vmsInFolder {
// Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted
if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) {
vmRefs = append(vmRefs, vm.Reference())
}
}

var vmRefsToAdd []types.ManagedObjectReference
for _, vm := range vmRefs {
found := false
for _, existingVM := range vmGroup.Vm {
if existingVM.Value == vm.Value {
log.Debugf("VM %s already in VM group %s", machine.Name, config.VMGroup)
found = true
break
}
}
if !found {
vmRefsToAdd = append(vmRefsToAdd, vm)
}
}

// Add the VM to the VM group
vmGroup.Vm = append(vmGroup.Vm, vmRefsToAdd...)
cluster, err := session.Finder.ClusterComputeResource(ctx, config.Cluster)
if err != nil {
return err
}

spec := &types.ClusterConfigSpecEx{
GroupSpec: []types.ClusterGroupSpec{
{
ArrayUpdateSpec: types.ArrayUpdateSpec{
Operation: types.ArrayUpdateOperationEdit,
},
Info: vmGroup,
},
},
}

log.Debugf("Adding VM %s in VM group %s", machine.Name, config.VMGroup)
task, err := cluster.Reconfigure(ctx, spec, true)
if err != nil {
return err
}

taskResult, err := task.WaitForResultEx(ctx)
if err != nil {
return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name())
}
if taskResult.State != types.TaskInfoStateSuccess {
return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name())
}
log.Debugf("Successfully added VM %s in VM group %s", machine.Name, config.VMGroup)
return nil
}

func findVMGroup(ctx context.Context, session *Session, clusterName, vmGroup string) (*types.ClusterVmGroup, error) {
cluster, err := session.Finder.ClusterComputeResource(ctx, clusterName)
if err != nil {
return nil, err
}

clusterConfigInfoEx, err := cluster.Configuration(ctx)
if err != nil {
return nil, err
}

for _, group := range clusterConfigInfoEx.Group {
if clusterVMGroup, ok := group.(*types.ClusterVmGroup); ok {
if clusterVMGroup.Name == vmGroup {
return clusterVMGroup, nil
}
}
}
return nil, fmt.Errorf("cannot find VM group %s", vmGroup)
}

0 comments on commit 3ead298

Please sign in to comment.