diff --git a/pkg/asset/machines/clusterapi.go b/pkg/asset/machines/clusterapi.go index d85c718c5d4..b2073a46223 100644 --- a/pkg/asset/machines/clusterapi.go +++ b/pkg/asset/machines/clusterapi.go @@ -22,12 +22,14 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/machines/aws" + "github.com/openshift/installer/pkg/asset/machines/gcp" "github.com/openshift/installer/pkg/asset/manifests/capiutils" "github.com/openshift/installer/pkg/asset/rhcos" "github.com/openshift/installer/pkg/clusterapi" awstypes "github.com/openshift/installer/pkg/types/aws" awsdefaults "github.com/openshift/installer/pkg/types/aws/defaults" azuretypes "github.com/openshift/installer/pkg/types/azure" + gcptypes "github.com/openshift/installer/pkg/types/gcp" ) var _ asset.WritableRuntimeAsset = (*ClusterAPI)(nil) @@ -217,6 +219,37 @@ func (c *ClusterAPI) Generate(dependencies asset.Parents) error { }) case azuretypes.Name: // TODO: implement + case gcptypes.Name: + // Generate GCP master machines using ControPlane machinepool + mpool := defaultGCPMachinePoolPlatform(pool.Architecture) + mpool.Set(ic.Platform.GCP.DefaultMachinePlatform) + mpool.Set(pool.Platform.GCP) + pool.Platform.GCP = &mpool + + gcpMachines, err := gcp.GenerateMachines( + installConfig, + clusterID.InfraID, + &pool, + string(*rhcosImage), + ) + if err != nil { + return fmt.Errorf("failed to create master machine objects %w", err) + } + c.FileList = append(c.FileList, gcpMachines...) + + // Generate GCP bootstrap machines + bootstrapMachines, err := gcp.GenerateBootstrapMachines( + capiutils.GenerateBoostrapMachineName(clusterID.InfraID), + installConfig, + clusterID.InfraID, + &pool, + string(*rhcosImage), + ) + if err != nil { + return fmt.Errorf("failed to create bootstrap machine objects %w", err) + } + c.FileList = append(c.FileList, bootstrapMachines...) + default: // TODO: support other platforms } diff --git a/pkg/asset/machines/gcp/gcpmachines.go b/pkg/asset/machines/gcp/gcpmachines.go new file mode 100644 index 00000000000..a28087590c0 --- /dev/null +++ b/pkg/asset/machines/gcp/gcpmachines.go @@ -0,0 +1,182 @@ +// Package gcp generates Machine objects for gcp. +package gcp + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + capg "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + capi "sigs.k8s.io/cluster-api/api/v1beta1" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/types" + gcptypes "github.com/openshift/installer/pkg/types/gcp" +) + +// GenerateMachines returns manifests and runtime objects to provision control plane nodes using CAPI. +func GenerateMachines(installConfig *installconfig.InstallConfig, infraID string, pool *types.MachinePool, imageName string) ([]*asset.RuntimeFile, error) { + var result []*asset.RuntimeFile + if poolPlatform := pool.Platform.Name(); poolPlatform != gcptypes.Name { + return nil, fmt.Errorf("non-GCP machine-pool: %q", poolPlatform) + } + mpool := pool.Platform.GCP + + total := int64(1) + if pool.Replicas != nil { + total = *pool.Replicas + } + + // Create GCP and CAPI machines for all master replicas in pool + for idx := int64(0); idx < total; idx++ { + name := fmt.Sprintf("%s-%s-%d", infraID, pool.Name, idx) + gcpMachine := createGCPMachine(name, installConfig, infraID, mpool, imageName) + + result = append(result, &asset.RuntimeFile{ + File: asset.File{Filename: fmt.Sprintf("10_inframachine_%s.yaml", gcpMachine.Name)}, + Object: gcpMachine, + }) + + dataSecret := fmt.Sprintf("%s-master", infraID) + capiMachine := createCAPIMachine(gcpMachine.Name, dataSecret, infraID) + + result = append(result, &asset.RuntimeFile{ + File: asset.File{Filename: fmt.Sprintf("10_machine_%s.yaml", capiMachine.Name)}, + Object: capiMachine, + }) + } + return result, nil +} + +// GenerateBootstrapMachines returns a manifest and runtime object for a bootstrap node using CAPI. +func GenerateBootstrapMachines(name string, installConfig *installconfig.InstallConfig, infraID string, pool *types.MachinePool, imageName string) ([]*asset.RuntimeFile, error) { + var result []*asset.RuntimeFile + if poolPlatform := pool.Platform.Name(); poolPlatform != gcptypes.Name { + return nil, fmt.Errorf("non-GCP machine-pool: %q", poolPlatform) + } + mpool := pool.Platform.GCP + + // Create one GCP and CAPI machine for bootstrap + bootstrapGCPMachine := createGCPMachine(name, installConfig, infraID, mpool, imageName) + + // Identify this as a bootstrap machine + bootstrapGCPMachine.Labels["install.openshift.io/bootstrap"] = "" + + result = append(result, &asset.RuntimeFile{ + File: asset.File{Filename: fmt.Sprintf("10_inframachine_%s.yaml", bootstrapGCPMachine.Name)}, + Object: bootstrapGCPMachine, + }) + + dataSecret := fmt.Sprintf("%s-%s", infraID, "bootstrap") + bootstrapCapiMachine := createCAPIMachine(bootstrapGCPMachine.Name, dataSecret, infraID) + + result = append(result, &asset.RuntimeFile{ + File: asset.File{Filename: fmt.Sprintf("10_machine_%s.yaml", bootstrapCapiMachine.Name)}, + Object: bootstrapCapiMachine, + }) + return result, nil +} + +// Create a CAPG-specific machine. +func createGCPMachine(name string, installConfig *installconfig.InstallConfig, infraID string, mpool *gcptypes.MachinePool, imageName string) *capg.GCPMachine { + // Use the rhcosImage as image name if not defined + var osImage string + if mpool.OSImage == nil { + osImage = imageName + } else { + osImage = mpool.OSImage.Name + } + + // TODO tags aren't currently being set in GCPMachine which only has + // AdditionalNetworkTags []string + + masterSubnet := installConfig.Config.Platform.GCP.ControlPlaneSubnet + if masterSubnet == "" { + masterSubnet = gcptypes.DefaultSubnetName(infraID, "master") + } + + gcpMachine := &capg.GCPMachine{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "GCPMachine", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "cluster.x-k8s.io/control-plane": "", + }, + }, + Spec: capg.GCPMachineSpec{ + InstanceType: mpool.InstanceType, + Subnet: ptr.To(masterSubnet), + AdditionalLabels: getLabelsFromInstallConfig(installConfig, infraID), + Image: ptr.To(osImage), + RootDeviceType: ptr.To(capg.DiskType(mpool.OSDisk.DiskType)), + RootDeviceSize: mpool.OSDisk.DiskSizeGB, + }, + } + // Set optional values from machinepool + if mpool.OnHostMaintenance != "" { + gcpMachine.Spec.OnHostMaintenance = ptr.To(capg.HostMaintenancePolicy(mpool.OnHostMaintenance)) + } + if mpool.ConfidentialCompute != "" { + gcpMachine.Spec.ConfidentialCompute = ptr.To(capg.ConfidentialComputePolicy(mpool.ConfidentialCompute)) + } + if mpool.SecureBoot != "" { + shieldedInstanceConfig := capg.GCPShieldedInstanceConfig{} + shieldedInstanceConfig.SecureBoot = capg.SecureBootPolicyEnabled + gcpMachine.Spec.ShieldedInstanceConfig = ptr.To(shieldedInstanceConfig) + } + if mpool.ServiceAccount != "" { + serviceAccount := &capg.ServiceAccount{ + Email: mpool.ServiceAccount, + // Set scopes to value defined at + // https://cloud.google.com/compute/docs/access/service-accounts#scopes_best_practice + Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"}, + } + gcpMachine.Spec.ServiceAccount = serviceAccount + } + + return gcpMachine +} + +// Create a CAPI machine based on the CAPG machine. +func createCAPIMachine(name string, dataSecret string, infraID string) *capi.Machine { + machine := &capi.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "cluster.x-k8s.io/control-plane": "", + }, + }, + Spec: capi.MachineSpec{ + ClusterName: infraID, + // Leave empty until ignition support is added + // Bootstrap: capi.Bootstrap{ + // DataSecretName: ptr.To(dataSecret), + // }, + InfrastructureRef: v1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "GCPMachine", + Name: name, + }, + }, + } + + return machine +} + +func getLabelsFromInstallConfig(installConfig *installconfig.InstallConfig, infraID string) map[string]string { + ic := installConfig.Config + + userLabels := map[string]string{} + for _, label := range ic.Platform.GCP.UserLabels { + userLabels[label.Key] = label.Value + } + // add OCP default label + userLabels[fmt.Sprintf("kubernetes-io-cluster-%s", infraID)] = "owned" + + return userLabels +} diff --git a/pkg/asset/machines/gcp/gcpmachines_test.go b/pkg/asset/machines/gcp/gcpmachines_test.go new file mode 100644 index 00000000000..928563c9fee --- /dev/null +++ b/pkg/asset/machines/gcp/gcpmachines_test.go @@ -0,0 +1,226 @@ +// Package gcp generates Machine objects for gcp. +package gcp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + capg "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + capi "sigs.k8s.io/cluster-api/api/v1beta1" + + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/types" + gcptypes "github.com/openshift/installer/pkg/types/gcp" +) + +const ( + numReplicas = 3 +) + +func Test_GenerateMachines(t *testing.T) { + cases := []struct { + name string + installConfig *installconfig.InstallConfig + expectedGCPConfig *capg.GCPMachine + expectedError string + }{ + { + name: "base configuration", + installConfig: getBaseInstallConfig(), + expectedGCPConfig: getBaseGCPMachine(), + }, + { + name: "additional labels", + installConfig: getICWithLabels(), + expectedGCPConfig: getGCPMachineWithLabels(), + }, + { + name: "onhostmaintenance", + installConfig: getICWithOnHostMaintenance(), + expectedGCPConfig: getGCPMachineWithOnHostMaintenance(), + }, + { + name: "confidentialcompute", + installConfig: getICWithConfidentialCompute(), + expectedGCPConfig: getGCPMachineWithConfidentialCompute(), + }, + { + name: "secureboot", + installConfig: getICWithSecureBoot(), + expectedGCPConfig: getGCPMachineWithSecureBoot(), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + installConfig := tc.installConfig + ic := installConfig.Config + pool := ic.ControlPlane + infraID := "012345678" + rhcosImage := "rhcos-415-92-202311241643-0-gcp-x86-64" + + mpool := gcptypes.MachinePool{ + InstanceType: "n2-standard-4", + OSDisk: gcptypes.OSDisk{ + DiskSizeGB: 128, + DiskType: "pd-ssd", + }, + } + mpool.Set(ic.Platform.GCP.DefaultMachinePlatform) + mpool.Set(pool.Platform.GCP) + pool.Platform.GCP = &mpool + + gcpMachines, err := GenerateMachines( + installConfig, + infraID, + pool, + rhcosImage, + ) + + if tc.expectedError != "" { + assert.Equal(t, tc.expectedError, err.Error()) + } else { + assert.NoError(t, err) + assert.NotEmpty(t, gcpMachines) + + assert.Equal(t, numReplicas*2, len(gcpMachines)) + // Check first set of GCP and CAPI machines + actualGCPMachine := gcpMachines[0].Object + actualCapiMachine := gcpMachines[1].Object + assert.Equal(t, tc.expectedGCPConfig, actualGCPMachine) + assert.Equal(t, getBaseCapiMachine(), actualCapiMachine) + } + }) + } +} + +func getBaseInstallConfig() *installconfig.InstallConfig { + return &installconfig.InstallConfig{ + AssetBase: installconfig.AssetBase{ + Config: &types.InstallConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ocp-edge-cluster-0", + Namespace: "cluster-0", + }, + BaseDomain: "testing.com", + ControlPlane: &types.MachinePool{ + Name: "master", + Replicas: ptr.To(int64(numReplicas)), + Platform: types.MachinePoolPlatform{}, + }, + Platform: types.Platform{ + GCP: &gcptypes.Platform{ + ProjectID: "my-project", + Region: "us-east1", + }, + }, + }, + }, + } +} + +func getICWithLabels() *installconfig.InstallConfig { + ic := getBaseInstallConfig() + ic.Config.Platform.GCP.UserLabels = []gcptypes.UserLabel{{Key: "foo", Value: "bar"}, + {Key: "id", Value: "1234"}} + return ic +} + +func getICWithOnHostMaintenance() *installconfig.InstallConfig { + ic := getBaseInstallConfig() + ic.Config.Platform.GCP.DefaultMachinePlatform = &gcptypes.MachinePool{OnHostMaintenance: "Terminate"} + return ic +} + +func getICWithConfidentialCompute() *installconfig.InstallConfig { + ic := getBaseInstallConfig() + ic.Config.Platform.GCP.DefaultMachinePlatform = &gcptypes.MachinePool{ConfidentialCompute: "Enabled"} + return ic +} + +func getICWithSecureBoot() *installconfig.InstallConfig { + ic := getBaseInstallConfig() + ic.Config.Platform.GCP.DefaultMachinePlatform = &gcptypes.MachinePool{SecureBoot: "Enabled"} + return ic +} + +func getBaseGCPMachine() *capg.GCPMachine { + subnet := "012345678-master-subnet" + image := "rhcos-415-92-202311241643-0-gcp-x86-64" + diskType := "pd-ssd" + gcpMachine := &capg.GCPMachine{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "GCPMachine", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "012345678-master-0", + Labels: map[string]string{ + "cluster.x-k8s.io/control-plane": "", + }, + }, + Spec: capg.GCPMachineSpec{ + InstanceType: "n2-standard-4", + Subnet: &subnet, + Image: &image, + AdditionalLabels: capg.Labels{ + "kubernetes-io-cluster-012345678": "owned", + }, + RootDeviceSize: 128, + RootDeviceType: ptr.To(capg.DiskType(diskType)), + }, + } + return gcpMachine +} + +func getGCPMachineWithLabels() *capg.GCPMachine { + gcpMachine := getBaseGCPMachine() + gcpMachine.Spec.AdditionalLabels = capg.Labels{ + "kubernetes-io-cluster-012345678": "owned", + "foo": "bar", + "id": "1234"} + return gcpMachine +} + +func getGCPMachineWithOnHostMaintenance() *capg.GCPMachine { + gcpMachine := getBaseGCPMachine() + var maint capg.HostMaintenancePolicy = "Terminate" + gcpMachine.Spec.OnHostMaintenance = &maint + return gcpMachine +} + +func getGCPMachineWithConfidentialCompute() *capg.GCPMachine { + gcpMachine := getBaseGCPMachine() + var cc capg.ConfidentialComputePolicy = "Enabled" + gcpMachine.Spec.ConfidentialCompute = &cc + return gcpMachine +} + +func getGCPMachineWithSecureBoot() *capg.GCPMachine { + gcpMachine := getBaseGCPMachine() + secureBoot := capg.GCPShieldedInstanceConfig{SecureBoot: capg.SecureBootPolicy("Enabled")} + gcpMachine.Spec.ShieldedInstanceConfig = &secureBoot + return gcpMachine +} + +func getBaseCapiMachine() *capi.Machine { + capiMachine := &capi.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "012345678-master-0", + Labels: map[string]string{ + "cluster.x-k8s.io/control-plane": "", + }, + }, + Spec: capi.MachineSpec{ + ClusterName: "012345678", + InfrastructureRef: v1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "GCPMachine", + Name: "012345678-master-0", + }, + }, + } + return capiMachine +} diff --git a/pkg/types/gcp/platform.go b/pkg/types/gcp/platform.go index 568a23e8625..c093e2f02f8 100644 --- a/pkg/types/gcp/platform.go +++ b/pkg/types/gcp/platform.go @@ -1,5 +1,9 @@ package gcp +import ( + "fmt" +) + // UserProvisionedDNS indicates whether the DNS solution is provisioned by the Installer or the user. type UserProvisionedDNS string @@ -105,3 +109,8 @@ type UserTag struct { // special characters `_-.@%=+:,*#&(){}[]` and spaces. Value string `json:"value"` } + +// DefaultSubnetName sets a default name for the subnet. +func DefaultSubnetName(infraID, role string) string { + return fmt.Sprintf("%s-%s-subnet", infraID, role) +}