Add an AgentBaker interface for RP to use. (Azure#926)

* Add an AgentBaker interface for RP to use. This is the preparation step to separate AgentBaker into a standalone service. RP will call the API provided by AgentBaker to get CustomData, CSE, and VHD image to provision nodes. We'll use protocol buffer and gRPC for the API definition. For now, use straight Go interface as API to help testing the end to end flow. * Return error if cloud is not found in the OS image settings. * Remove setting for CoreOS. It's not used.
UtheMan · Jul 13, 2021 · fdbf9c0 · fdbf9c0
1 parent a560247
commit fdbf9c0
Show file tree

Hide file tree

Showing 5 changed files with 335 additions and 0 deletions.
diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+package agent
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
+)
+
+type AgentBaker interface {
+	GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error)
+}
+
+func NewAgentBaker() (AgentBaker, error) {
+	return &agentBakerImpl{}, nil
+}
+
+type agentBakerImpl struct{}
+
+func (agentBaker *agentBakerImpl) GetNodeBootstrapping(ctx context.Context,
+	config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) {
+	templateGenerator := InitializeTemplateGenerator()
+	nodeBootstrapping := &datamodel.NodeBootstrapping{
+		CustomData: templateGenerator.GetNodeBootstrappingPayload(config),
+		CSE:        templateGenerator.GetNodeBootstrappingCmd(config),
+	}
+
+	osImageConfigMap, hasCloud := datamodel.AzureCloudToOSImageMap[config.CloudSpecConfig.CloudName]
+	if !hasCloud {
+		return nil, fmt.Errorf("don't have settings for cloud %s", config.CloudSpecConfig.CloudName)
+	}
+
+	if osImageConfig, hasImage := osImageConfigMap[config.AgentPoolProfile.Distro]; hasImage {
+		nodeBootstrapping.OSImageConfig = osImageConfig
+	}
+
+	return nodeBootstrapping, nil
+}
diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go
@@ -0,0 +1,151 @@
+package agent
+
+import (
+	"context"
+
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("GetNodeBootstrapping", func() {
+	var (
+		cs     *datamodel.ContainerService
+		config *datamodel.NodeBootstrappingConfiguration
+	)
+
+	BeforeEach(func() {
+		cs = &datamodel.ContainerService{
+			Location: "southcentralus",
+			Type:     "Microsoft.ContainerService/ManagedClusters",
+			Properties: &datamodel.Properties{
+				OrchestratorProfile: &datamodel.OrchestratorProfile{
+					OrchestratorType:    datamodel.Kubernetes,
+					OrchestratorVersion: "1.16.15",
+					KubernetesConfig:    &datamodel.KubernetesConfig{},
+				},
+				HostedMasterProfile: &datamodel.HostedMasterProfile{
+					DNSPrefix: "uttestdom",
+				},
+				AgentPoolProfiles: []*datamodel.AgentPoolProfile{
+					{
+						Name:                "agent2",
+						VMSize:              "Standard_DS1_v2",
+						StorageProfile:      "ManagedDisks",
+						OSType:              datamodel.Linux,
+						VnetSubnetID:        "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/aks-vnet-07752737/subnet/subnet1",
+						AvailabilityProfile: datamodel.VirtualMachineScaleSets,
+						Distro:              datamodel.AKSUbuntu1604,
+					},
+				},
+				LinuxProfile: &datamodel.LinuxProfile{
+					AdminUsername: "azureuser",
+				},
+				ServicePrincipalProfile: &datamodel.ServicePrincipalProfile{
+					ClientID: "ClientID",
+					Secret:   "Secret",
+				},
+			},
+		}
+		cs.Properties.LinuxProfile.SSH.PublicKeys = []datamodel.PublicKey{{
+			KeyData: string("testsshkey"),
+		}}
+
+		agentPool := cs.Properties.AgentPoolProfiles[0]
+
+		fullK8sComponentsMap := K8sComponentsByVersionMap[cs.Properties.OrchestratorProfile.OrchestratorVersion]
+		pauseImage := cs.Properties.OrchestratorProfile.KubernetesConfig.MCRKubernetesImageBase + fullK8sComponentsMap["pause"]
+
+		hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
+		hyperkubeImage := hyperkubeImageBase + fullK8sComponentsMap["hyperkube"]
+		if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {
+			hyperkubeImage = cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage
+		}
+
+		windowsPackage := datamodel.AzurePublicCloudSpecForTest.KubernetesSpecConfig.KubeBinariesSASURLBase + fullK8sComponentsMap["windowszip"]
+		k8sComponents := &datamodel.K8sComponents{
+			PodInfraContainerImageURL: pauseImage,
+			HyperkubeImageURL:         hyperkubeImage,
+			WindowsPackageURL:         windowsPackage,
+		}
+
+		kubeletConfig := map[string]string{
+			"--address":                           "0.0.0.0",
+			"--pod-manifest-path":                 "/etc/kubernetes/manifests",
+			"--cloud-provider":                    "azure",
+			"--cloud-config":                      "/etc/kubernetes/azure.json",
+			"--azure-container-registry-config":   "/etc/kubernetes/azure.json",
+			"--cluster-domain":                    "cluster.local",
+			"--cluster-dns":                       "10.0.0.10",
+			"--cgroups-per-qos":                   "true",
+			"--tls-cert-file":                     "/etc/kubernetes/certs/kubeletserver.crt",
+			"--tls-private-key-file":              "/etc/kubernetes/certs/kubeletserver.key",
+			"--tls-cipher-suites":                 "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256",
+			"--max-pods":                          "110",
+			"--node-status-update-frequency":      "10s",
+			"--image-gc-high-threshold":           "85",
+			"--image-gc-low-threshold":            "80",
+			"--event-qps":                         "0",
+			"--pod-max-pids":                      "-1",
+			"--enforce-node-allocatable":          "pods",
+			"--streaming-connection-idle-timeout": "4h0m0s",
+			"--rotate-certificates":               "true",
+			"--read-only-port":                    "10255",
+			"--protect-kernel-defaults":           "true",
+			"--resolv-conf":                       "/etc/resolv.conf",
+			"--anonymous-auth":                    "false",
+			"--client-ca-file":                    "/etc/kubernetes/certs/ca.crt",
+			"--authentication-token-webhook":      "true",
+			"--authorization-mode":                "Webhook",
+			"--eviction-hard":                     "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%",
+			"--feature-gates":                     "RotateKubeletServerCertificate=true,a=b,PodPriority=true,x=y",
+			"--system-reserved":                   "cpu=2,memory=1Gi",
+			"--kube-reserved":                     "cpu=100m,memory=1638Mi",
+		}
+
+		config = &datamodel.NodeBootstrappingConfiguration{
+			ContainerService:              cs,
+			CloudSpecConfig:               datamodel.AzurePublicCloudSpecForTest,
+			K8sComponents:                 k8sComponents,
+			AgentPoolProfile:              agentPool,
+			TenantID:                      "tenantID",
+			SubscriptionID:                "subID",
+			ResourceGroupName:             "resourceGroupName",
+			UserAssignedIdentityClientID:  "userAssignedID",
+			ConfigGPUDriverIfNeeded:       true,
+			EnableGPUDevicePluginIfNeeded: false,
+			EnableKubeletConfigFile:       false,
+			EnableNvidia:                  false,
+			FIPSEnabled:                   false,
+			KubeletConfig:                 kubeletConfig,
+			PrimaryScaleSetName:           "aks-agent2-36873793-vmss",
+		}
+	})
+
+	It("should return correct boot strapping data", func() {
+		agentBaker, err := NewAgentBaker()
+		Expect(err).NotTo(HaveOccurred())
+
+		nodeBootStrapping, err := agentBaker.GetNodeBootstrapping(context.Background(), config)
+		Expect(err).NotTo(HaveOccurred())
+
+		// baker_test.go tested the correctness of the generated Custom Data and CSE, so here
+		// we just do a sanity check of them not being empty.
+		Expect(nodeBootStrapping.CustomData).NotTo(Equal(""))
+		Expect(nodeBootStrapping.CSE).NotTo(Equal(""))
+
+		Expect(nodeBootStrapping.OSImageConfig.ImageOffer).To(Equal("aks"))
+		Expect(nodeBootStrapping.OSImageConfig.ImageSku).To(Equal("aks-ubuntu-1604-2021-q3"))
+		Expect(nodeBootStrapping.OSImageConfig.ImagePublisher).To(Equal("microsoft-aks"))
+		Expect(nodeBootStrapping.OSImageConfig.ImageVersion).To(Equal("2021.07.10"))
+	})
+
+	It("should return an error if cloud is not found", func() {
+		config.CloudSpecConfig.CloudName = "UnknownCloud"
+		agentBaker, err := NewAgentBaker()
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = agentBaker.GetNodeBootstrapping(context.Background(), config)
+		Expect(err).To(HaveOccurred())
+	})
+})
diff --git a/pkg/agent/datamodel/const.go b/pkg/agent/datamodel/const.go
@@ -68,8 +68,14 @@ const (
 )
 
 const (
+	// AzurePublicCloud is a const string reference identifier for public cloud
+	AzurePublicCloud = "AzurePublicCloud"
 	// AzureChinaCloud is a const string reference identifier for china cloud
 	AzureChinaCloud = "AzureChinaCloud"
+	// AzureGermanCloud is a const string reference identifier for german cloud
+	AzureGermanCloud = "AzureGermanCloud"
+	// AzureUSGovernmentCloud is a const string reference identifier for us government cloud
+	AzureUSGovernmentCloud = "AzureUSGovernmentCloud"
 	// AzureStackCloud is a const string reference identifier for Azure Stack cloud
 	AzureStackCloud = "AzureStackCloud"
 )

diff --git a/pkg/agent/datamodel/osimageconfig.go b/pkg/agent/datamodel/osimageconfig.go
@@ -0,0 +1,125 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+package datamodel
+
+var (
+	Ubuntu1604OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "UbuntuServer",
+		ImageSku:       "16.04-LTS",
+		ImagePublisher: "Canonical",
+		ImageVersion:   "latest",
+	}
+
+	Ubuntu1804OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "UbuntuServer",
+		ImageSku:       "18.04-LTS",
+		ImagePublisher: "Canonical",
+		ImageVersion:   "latest",
+	}
+
+	Ubuntu1804Gen2OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "aks",
+		ImageSku:       "aks-ubuntu-1804-gen2-2021-q3",
+		ImagePublisher: "microsoft-aks",
+		ImageVersion:   "2021.07.10",
+	}
+
+	RHELOSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "RHEL",
+		ImageSku:       "7.3",
+		ImagePublisher: "RedHat",
+		ImageVersion:   "latest",
+	}
+
+	AKSUbuntu1604OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "aks",
+		ImageSku:       "aks-ubuntu-1604-2021-q3",
+		ImagePublisher: "microsoft-aks",
+		ImageVersion:   "2021.07.10",
+	}
+
+	AKSUbuntu1804OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "aks",
+		ImageSku:       "aks-ubuntu-1804-2021-q3",
+		ImagePublisher: "microsoft-aks",
+		ImageVersion:   "2021.07.10",
+	}
+
+	AKSWindowsServer2019OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "aks-windows",
+		ImageSku:       "aks-2019-datacenter-core-smalldisk-2106",
+		ImagePublisher: "microsoft-aks",
+		ImageVersion:   "17763.1999.210609",
+	}
+
+	ACC1604OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "confidential-compute-preview",
+		ImageSku:       "16.04-LTS",
+		ImagePublisher: "Canonical",
+		ImageVersion:   "latest",
+	}
+
+	AKSUbuntuContainerd1804OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "aks-aez",
+		ImageSku:       "aks-ubuntu-containerd-1804-2021-q2",
+		ImagePublisher: "microsoft-aks",
+		ImageVersion:   "2021.04.27",
+	}
+
+	AKSUbuntuContainerd1804Gen2OSImageConfig = AzureOSImageConfig{
+		ImageOffer:     "aks-aez",
+		ImageSku:       "aks-ubuntu-containerd-1804-gen2-2021-q2",
+		ImagePublisher: "microsoft-aks",
+		ImageVersion:   "2021.05.01",
+	}
+
+	AzureCloudToOSImageMap = map[string]map[Distro]AzureOSImageConfig{
+		AzureChinaCloud: {
+			Ubuntu:            Ubuntu1604OSImageConfig,
+			Ubuntu1804:        Ubuntu1804OSImageConfig,
+			Ubuntu1804Gen2:    Ubuntu1804Gen2OSImageConfig,
+			RHEL:              RHELOSImageConfig,
+			AKSUbuntu1604:     AKSUbuntu1604OSImageConfig,
+			AKS1604Deprecated: AKSUbuntu1604OSImageConfig, // for back-compat
+			AKSUbuntu1804:     AKSUbuntu1804OSImageConfig,
+			AKS1804Deprecated: AKSUbuntu1804OSImageConfig, // for back-compat
+			AKSWindows2019PIR: AKSWindowsServer2019OSImageConfig,
+		},
+		AzureGermanCloud: {
+			Ubuntu:            Ubuntu1604OSImageConfig,
+			Ubuntu1804:        Ubuntu1804OSImageConfig,
+			Ubuntu1804Gen2:    Ubuntu1804Gen2OSImageConfig,
+			RHEL:              RHELOSImageConfig,
+			AKSUbuntu1604:     Ubuntu1604OSImageConfig,
+			AKS1604Deprecated: Ubuntu1604OSImageConfig, // for back-compat
+			AKSUbuntu1804:     Ubuntu1604OSImageConfig, // workaround for https://github.com/Azure/aks-engine/issues/761
+			AKS1804Deprecated: Ubuntu1604OSImageConfig, // for back-compat
+			AKSWindows2019PIR: AKSWindowsServer2019OSImageConfig,
+		},
+		AzureUSGovernmentCloud: {
+			Ubuntu:            Ubuntu1604OSImageConfig,
+			Ubuntu1804:        Ubuntu1804OSImageConfig,
+			Ubuntu1804Gen2:    Ubuntu1804Gen2OSImageConfig,
+			RHEL:              RHELOSImageConfig,
+			AKSUbuntu1604:     AKSUbuntu1604OSImageConfig,
+			AKS1604Deprecated: AKSUbuntu1604OSImageConfig, // for back-compat
+			AKSUbuntu1804:     AKSUbuntu1804OSImageConfig,
+			AKS1804Deprecated: AKSUbuntu1804OSImageConfig, // for back-compat
+			AKSWindows2019PIR: AKSWindowsServer2019OSImageConfig,
+		},
+		AzurePublicCloud: {
+			Ubuntu:                      Ubuntu1604OSImageConfig,
+			Ubuntu1804:                  Ubuntu1804OSImageConfig,
+			Ubuntu1804Gen2:              Ubuntu1804Gen2OSImageConfig,
+			RHEL:                        RHELOSImageConfig,
+			AKSUbuntu1604:               AKSUbuntu1604OSImageConfig,
+			AKS1604Deprecated:           AKSUbuntu1604OSImageConfig, // for back-compat
+			AKSUbuntu1804:               AKSUbuntu1804OSImageConfig,
+			AKS1804Deprecated:           AKSUbuntu1804OSImageConfig, // for back-compat
+			AKSUbuntuContainerd1804:     AKSUbuntuContainerd1804OSImageConfig,
+			AKSUbuntuContainerd1804Gen2: AKSUbuntuContainerd1804Gen2OSImageConfig,
+			AKSWindows2019PIR:           AKSWindowsServer2019OSImageConfig,
+		},
+	}
+)
diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go
@@ -136,6 +136,11 @@ const (
 	AKSUbuntuFipsContainerd1804Gen2    Distro = "aks-ubuntu-fips-containerd-18.04-gen2"
 	AKSUbuntuFipsGPUContainerd1804     Distro = "aks-ubuntu-fips-gpu-containerd-18.04"
 	AKSUbuntuFipsGPUContainerd1804Gen2 Distro = "aks-ubuntu-fips-gpu-containerd-18.04-gen2"
+	RHEL                               Distro = "rhel"
+	CoreOS                             Distro = "coreos"
+	AKS1604Deprecated                  Distro = "aks"      // deprecated AKS 16.04 distro. Equivalent to aks-ubuntu-16.04.
+	AKS1804Deprecated                  Distro = "aks-1804" // deprecated AKS 18.04 distro. Equivalent to aks-ubuntu-18.04.
+	AKSWindows2019PIR                  Distro = "aks-windows-2019-pir"
 )
 
 var AKSDistrosAvailableOnVHD []Distro = []Distro{
@@ -1301,6 +1306,13 @@ type NodeBootstrappingConfiguration struct {
 	PrimaryScaleSetName            string
 }
 
+// NodeBootstrapping represents the custom data, CSE, and OS image info needed for node bootstrapping.
+type NodeBootstrapping struct {
+	CustomData    string
+	CSE           string
+	OSImageConfig AzureOSImageConfig
+}
+
 // HTTPProxyConfig represents configurations of http proxy
 type HTTPProxyConfig struct {
 	HTTPProxy  *string   `json:"httpProxy,omitempty"`