feat: add arbiter node role assets and logic

added arbiter node role assets that mirror master, this will be changed before GA
added logic to handle arbiter node assets only when arbiter node is explicitly defined

Signed-off-by: ehila <ehila@redhat.com>

Author: eggfoobar

manifests/arbiter.machineconfigpool.yaml

@@ -0,0 +1,15 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfigPool
+metadata:
+  name: arbiter
+  labels:
+    "operator.machineconfiguration.openshift.io/required-for-upgrade": ""
+    "machineconfiguration.openshift.io/mco-built-in": ""
+    "pools.operator.machineconfiguration.openshift.io/arbiter": ""
+spec:
+  machineConfigSelector:
+    matchLabels:
+      "machineconfiguration.openshift.io/role": "arbiter"
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/arbiter: ""

manifests/machineconfigcontroller/custom-machine-config-pool-selector-validatingadmissionpolicy.yaml

@@ -22,6 +22,8 @@ spec:
             (object.spec.machineConfigSelector.matchLabels["machineconfiguration.openshift.io/role"] == "master")
             ||
             (object.spec.machineConfigSelector.matchLabels["machineconfiguration.openshift.io/role"] == "worker")
+            ||
+            (object.spec.machineConfigSelector.matchLabels["machineconfiguration.openshift.io/role"] == "arbiter")
           )
         )
         ||

pkg/controller/common/constants.go

@@ -52,6 +52,9 @@ const (
 	// APIServerInstanceName is a singleton name for APIServer configuration
 	APIServerBootstrapFileLocation = "/etc/mcs/bootstrap/api-server/api-server.yaml"
 
+	// MachineConfigPoolArbiter is the MachineConfigPool name given to the arbiter
+	MachineConfigPoolArbiter = "arbiter"
+
 	// MachineConfigPoolMaster is the MachineConfigPool name given to the master
 	MachineConfigPoolMaster = "master"
 

pkg/controller/kubelet-config/kubelet_config_nodes.go

@@ -133,7 +133,8 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error {
 			}
 		}
 		// The following code updates the MC with the relevant CGroups version
-		if role == ctrlcommon.MachineConfigPoolWorker || role == ctrlcommon.MachineConfigPoolMaster {
+		switch role {
+		case ctrlcommon.MachineConfigPoolWorker, ctrlcommon.MachineConfigPoolMaster, ctrlcommon.MachineConfigPoolArbiter:
 			err = updateMachineConfigwithCgroup(nodeConfig, mc)
 			if err != nil {
 				return err

pkg/controller/template/render.go

@@ -47,6 +47,9 @@ const (
 	platformBase   = "_base"
 	platformOnPrem = "on-prem"
 	sno            = "sno"
+	masterRole     = "master"
+	workerRole     = "worker"
+	arbiterRole    = "arbiter"
 )
 
 // generateTemplateMachineConfigs returns MachineConfig objects from the templateDir and a config object
@@ -80,6 +83,11 @@ func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([
 			continue
 		}
 
+		// Avoid creating resources for non arbiter deployments
+		if role == arbiterRole && hasControlPlaneTopology(config, configv1.HighlyAvailableArbiterMode) {
+			continue
+		}
+
 		roleConfigs, err := GenerateMachineConfigsForRole(config, role, templateDir)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create MachineConfig for role %s: %w", role, err)
@@ -102,10 +110,10 @@ func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([
 func GenerateMachineConfigsForRole(config *RenderConfig, role, templateDir string) ([]*mcfgv1.MachineConfig, error) {
 	rolePath := role
 	//nolint:goconst
-	if role != "worker" && role != "master" {
+	if role != workerRole && role != masterRole && role != arbiterRole {
 		// custom pools are only allowed to be worker's children
 		// and can reuse the worker templates
-		rolePath = "worker"
+		rolePath = workerRole
 	}
 
 	path := filepath.Join(templateDir, rolePath)
@@ -219,7 +227,7 @@ func getPaths(config *RenderConfig, platformString string) []string {
 	platformBasedPaths = append(platformBasedPaths, platformString)
 
 	// sno is specific case and it should override even specific platform files
-	if config.Infra.Status.ControlPlaneTopology == configv1.SingleReplicaTopologyMode {
+	if hasControlPlaneTopology(config, configv1.SingleReplicaTopologyMode) {
 		platformBasedPaths = append(platformBasedPaths, sno)
 	}
 
@@ -799,3 +807,12 @@ func cloudPlatformLoadBalancerIPState(cfg RenderConfig) LoadBalancerIPState {
 	}
 	return lbIPState
 }
+
+// hasControlPlaneTopology returns true if the topology matches the infra.controlPlaneTopology
+// checks to make sure RenderConfig and Infra are not nil.
+func hasControlPlaneTopology(r *RenderConfig, topo configv1.TopologyMode) bool {
+	if r == nil || r.Infra == nil {
+		return false
+	}
+	return r.Infra.Status.ControlPlaneTopology == topo
+}

pkg/controller/template/render_test.go

@@ -203,6 +203,7 @@ var (
 	configs = map[string]string{
 		"aws":                    "./test_data/controller_config_aws.yaml",
 		"baremetal":              "./test_data/controller_config_baremetal.yaml",
+		"baremetal-arbiter":      "./test_data/controller_config_baremetal_arbiter.yaml",
 		"gcp":                    "./test_data/controller_config_gcp.yaml",
 		"openstack":              "./test_data/controller_config_openstack.yaml",
 		"libvirt":                "./test_data/controller_config_libvirt.yaml",
@@ -281,7 +282,7 @@ func TestGenerateMachineConfigs(t *testing.T) {
 			if err != nil {
 				t.Errorf("Failed to parse Ignition config for %s, %s, error: %v", config, cfg.Name, err)
 			}
-			if role == "master" {
+			if role == masterRole {
 				if !foundPullSecretMaster {
 					foundPullSecretMaster = findIgnFile(ign.Storage.Files, "/var/lib/kubelet/config.json", t)
 				}
@@ -292,7 +293,7 @@ func TestGenerateMachineConfigs(t *testing.T) {
 					foundMTUMigrationMaster = findIgnFile(ign.Storage.Files, "/usr/local/bin/mtu-migration.sh", t)
 					foundMTUMigrationMaster = foundMTUMigrationMaster || findIgnFile(ign.Storage.Files, "/etc/systemd/system/mtu-migration.service", t)
 				}
-			} else if role == "worker" {
+			} else if role == workerRole {
 				if !foundPullSecretWorker {
 					foundPullSecretWorker = findIgnFile(ign.Storage.Files, "/var/lib/kubelet/config.json", t)
 				}
@@ -303,6 +304,18 @@ func TestGenerateMachineConfigs(t *testing.T) {
 					foundMTUMigrationWorker = findIgnFile(ign.Storage.Files, "/usr/local/bin/mtu-migration.sh", t)
 					foundMTUMigrationWorker = foundMTUMigrationWorker || findIgnFile(ign.Storage.Files, "/etc/systemd/system/mtu-migration.service", t)
 				}
+			} else if role == arbiterRole {
+				// arbiter role currently follows master output
+				if !foundPullSecretMaster {
+					foundPullSecretMaster = findIgnFile(ign.Storage.Files, "/var/lib/kubelet/config.json", t)
+				}
+				if !foundKubeletUnitMaster {
+					foundKubeletUnitMaster = findIgnUnit(ign.Systemd.Units, "kubelet.service", t)
+				}
+				if !foundMTUMigrationMaster {
+					foundMTUMigrationMaster = findIgnFile(ign.Storage.Files, "/usr/local/bin/mtu-migration.sh", t)
+					foundMTUMigrationMaster = foundMTUMigrationMaster || findIgnFile(ign.Storage.Files, "/etc/systemd/system/mtu-migration.service", t)
+				}
 			} else {
 				t.Fatalf("Unknown role %s", role)
 			}

pkg/controller/template/test_data/controller_config_baremetal_arbiter.yaml

@@ -0,0 +1,37 @@
+apiVersion: "machineconfigurations.openshift.io/v1"
+kind: "ControllerConfig"
+spec:
+  clusterDNSIP: "10.3.0.10"
+  cloudProviderConfig: ""
+  etcdInitialCount: 3
+  etcdCAData: ZHVtbXkgZXRjZC1jYQo=
+  rootCAData: ZHVtbXkgcm9vdC1jYQo=
+  pullSecret:
+    data: ZHVtbXkgZXRjZC1jYQo=
+  images:
+    etcd: image/etcd:1
+    setupEtcdEnv: image/setupEtcdEnv:1
+    infraImage: image/infraImage:1
+    kubeClientAgentImage: image/kubeClientAgentImage:1
+  infra:
+    apiVersion: config.openshift.io/v1
+    kind: Infrastructure
+    spec:
+      cloudConfig:
+        key: config
+        name: cloud-provider-config
+    status:
+      apiServerInternalURI: https://api-int.my-test-cluster.installer.team.coreos.systems:6443
+      apiServerURL: https://api.my-test-cluster.installer.team.coreos.systems:6443
+      etcdDiscoveryDomain: my-test-cluster.installer.team.coreos.systems
+      infrastructureName: my-test-cluster
+      controlPlaneTopology: HighlyAvailableArbiter
+      platformStatus:
+        type: "BareMetal"
+        baremetal:
+          apiServerInternalIP: 10.0.0.1
+          ingressIP: 10.0.0.2
+          nodeDNSIP: 10.0.0.3
+  dns:
+    spec:
+      baseDomain: my-test-cluster.installer.team.coreos.systems

pkg/daemon/daemon.go

@@ -2728,6 +2728,8 @@ func (dn *Daemon) getControlPlaneTopology() configv1.TopologyMode {
 		return configv1.SingleReplicaTopologyMode
 	case configv1.HighlyAvailableTopologyMode:
 		return configv1.HighlyAvailableTopologyMode
+	case configv1.HighlyAvailableArbiterMode:
+		return configv1.HighlyAvailableArbiterMode
 	default:
 		// for any unhandled case, default to HighlyAvailableTopologyMode
 		return configv1.HighlyAvailableTopologyMode

pkg/operator/bootstrap.go

@@ -155,7 +155,7 @@ func RenderBootstrap(
 		templatectrl.KubeRbacProxyKey:       imgs.KubeRbacProxy,
 	}
 
-	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, nil, []*mcfgv1alpha1.MachineOSConfig{}, nil)
+	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra, nil, []*mcfgv1alpha1.MachineOSConfig{}, nil)
 
 	manifests := []manifest{
 		{
@@ -182,6 +182,13 @@ func RenderBootstrap(
 		},
 	}
 
+	if infra.Status.ControlPlaneTopology == configv1.HighlyAvailableArbiterMode {
+		manifests = append(manifests, manifest{
+			name:     "manifests/arbiter.machineconfigpool.yaml",
+			filename: "bootstrap/manifests/arbiter.machineconfigpool.yaml",
+		})
+	}
+
 	manifests = appendManifestsByPlatform(manifests, *infra)
 
 	for _, m := range manifests {

pkg/operator/sync.go

@@ -641,9 +641,9 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera
 		}
 
 		// create renderConfig
-		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, pointerConfigData, moscs, apiServer)
+		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra, pointerConfigData, moscs, apiServer)
 	} else {
-		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, pointerConfigData, nil, apiServer)
+		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra, pointerConfigData, nil, apiServer)
 	}
 
 	return nil
@@ -682,6 +682,11 @@ func (optr *Operator) syncMachineConfigPools(config *renderConfig, _ *configv1.C
 		"manifests/master.machineconfigpool.yaml",
 		"manifests/worker.machineconfigpool.yaml",
 	}
+
+	if config.Infra.Status.ControlPlaneTopology == configv1.HighlyAvailableArbiterMode {
+		mcps = append(mcps, "manifests/arbiter.machineconfigpool.yaml")
+	}
+
 	for _, mcp := range mcps {
 		mcpBytes, err := renderAsset(config, mcp)
 		if err != nil {
@@ -778,6 +783,8 @@ func (optr *Operator) syncMachineConfigNodes(_ *renderConfig, _ *configv1.Cluste
 			pool = "worker"
 		} else if _, ok = node.Labels["node-role.kubernetes.io/master"]; ok {
 			pool = "master"
+		} else if _, ok = node.Labels["node-role.kubernetes.io/arbiter"]; ok {
+			pool = "arbiter"
 		}
 		newMCS := &v1alpha1.MachineConfigNode{
 			Spec: v1alpha1.MachineConfigNodeSpec{
@@ -2035,7 +2042,7 @@ func setGVK(obj runtime.Object, scheme *runtime.Scheme) error {
 	return nil
 }
 
-func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *ctrlcommon.RenderConfigImages, apiServerURL string, pointerConfigData []byte, moscs []*mcfgv1alpha1.MachineOSConfig, apiServer *configv1.APIServer) *renderConfig {
+func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *ctrlcommon.RenderConfigImages, infra *configv1.Infrastructure, pointerConfigData []byte, moscs []*mcfgv1alpha1.MachineOSConfig, apiServer *configv1.APIServer) *renderConfig {
 	tlsMinVersion, tlsCipherSuites := ctrlcommon.GetSecurityProfileCiphersFromAPIServer(apiServer)
 	return &renderConfig{
 		TargetNamespace:        tnamespace,
@@ -2044,8 +2051,9 @@ func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.C
 		ControllerConfig:       *ccSpec,
 		Images:                 imgs,
 		KubeAPIServerServingCA: kubeAPIServerServingCA,
-		APIServerURL:           apiServerURL,
+		APIServerURL:           infra.Status.APIServerInternalURL,
 		PointerConfig:          string(pointerConfigData),
+		Infra:                  *infra,
 		MachineOSConfigs:       moscs,
 		TLSMinVersion:          tlsMinVersion,
 		TLSCipherSuites:        tlsCipherSuites,