OCPEDGE-1313: feat: added assets for arbiter node role

manifests/arbiter.machineconfigpool.yaml

@@ -0,0 +1,15 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfigPool
+metadata:
+  name: arbiter
+  labels:
+    "operator.machineconfiguration.openshift.io/required-for-upgrade": ""
+    "machineconfiguration.openshift.io/mco-built-in": ""
+    "pools.operator.machineconfiguration.openshift.io/arbiter": ""
+spec:
+  machineConfigSelector:
+    matchLabels:
+      "machineconfiguration.openshift.io/role": "arbiter"
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/arbiter: ""

manifests/machineconfigcontroller/custom-machine-config-pool-selector-validatingadmissionpolicy.yaml

@@ -22,6 +22,8 @@ spec:
             (object.spec.machineConfigSelector.matchLabels["machineconfiguration.openshift.io/role"] == "master")
             ||
             (object.spec.machineConfigSelector.matchLabels["machineconfiguration.openshift.io/role"] == "worker")
+            ||
+            (object.spec.machineConfigSelector.matchLabels["machineconfiguration.openshift.io/role"] == "arbiter")
           )
         )
         ||

pkg/controller/common/constants.go

@@ -52,6 +52,9 @@ const (
 	// APIServerInstanceName is a singleton name for APIServer configuration
 	APIServerBootstrapFileLocation = "/etc/mcs/bootstrap/api-server/api-server.yaml"
 
+	// MachineConfigPoolArbiter is the MachineConfigPool name given to the arbiter
+	MachineConfigPoolArbiter = "arbiter"
+
 	// MachineConfigPoolMaster is the MachineConfigPool name given to the master
 	MachineConfigPoolMaster = "master"
 

pkg/controller/kubelet-config/kubelet_config_nodes.go

@@ -133,7 +133,8 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error {
 			}
 		}
 		// The following code updates the MC with the relevant CGroups version
-		if role == ctrlcommon.MachineConfigPoolWorker || role == ctrlcommon.MachineConfigPoolMaster {
+		switch role {
+		case ctrlcommon.MachineConfigPoolWorker, ctrlcommon.MachineConfigPoolMaster, ctrlcommon.MachineConfigPoolArbiter:
 			err = updateMachineConfigwithCgroup(nodeConfig, mc)
 			if err != nil {
 				return err

pkg/controller/template/render.go

@@ -47,6 +47,9 @@ const (
 	platformBase   = "_base"
 	platformOnPrem = "on-prem"
 	sno            = "sno"
+	masterRole     = "master"
+	workerRole     = "worker"
+	arbiterRole    = "arbiter"
 )
 
 // generateTemplateMachineConfigs returns MachineConfig objects from the templateDir and a config object
@@ -80,6 +83,11 @@ func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([
 			continue
 		}
 
+		// Avoid creating resources for non arbiter deployments
+		if role == arbiterRole && !hasControlPlaneTopology(config, configv1.HighlyAvailableArbiterMode) {
+			continue
+		}
+
 		roleConfigs, err := GenerateMachineConfigsForRole(config, role, templateDir)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create MachineConfig for role %s: %w", role, err)
@@ -102,10 +110,10 @@ func generateTemplateMachineConfigs(config *RenderConfig, templateDir string) ([
 func GenerateMachineConfigsForRole(config *RenderConfig, role, templateDir string) ([]*mcfgv1.MachineConfig, error) {
 	rolePath := role
 	//nolint:goconst
-	if role != "worker" && role != "master" {
+	if role != workerRole && role != masterRole && role != arbiterRole {
 		// custom pools are only allowed to be worker's children
 		// and can reuse the worker templates
-		rolePath = "worker"
+		rolePath = workerRole
 	}
 
 	path := filepath.Join(templateDir, rolePath)
@@ -219,7 +227,7 @@ func getPaths(config *RenderConfig, platformString string) []string {
 	platformBasedPaths = append(platformBasedPaths, platformString)
 
 	// sno is specific case and it should override even specific platform files
-	if config.Infra.Status.ControlPlaneTopology == configv1.SingleReplicaTopologyMode {
+	if hasControlPlaneTopology(config, configv1.SingleReplicaTopologyMode) {
 		platformBasedPaths = append(platformBasedPaths, sno)
 	}
 
@@ -799,3 +807,12 @@ func cloudPlatformLoadBalancerIPState(cfg RenderConfig) LoadBalancerIPState {
 	}
 	return lbIPState
 }
+
+// hasControlPlaneTopology returns true if the topology matches the infra.controlPlaneTopology
+// checks to make sure RenderConfig and Infra are not nil.
+func hasControlPlaneTopology(r *RenderConfig, topo configv1.TopologyMode) bool {
+	if r == nil || r.Infra == nil {
+		return false
+	}
+	return r.Infra.Status.ControlPlaneTopology == topo
+}

pkg/controller/template/render_test.go

@@ -203,6 +203,7 @@ var (
 	configs = map[string]string{
 		"aws":                    "./test_data/controller_config_aws.yaml",
 		"baremetal":              "./test_data/controller_config_baremetal.yaml",
+		"baremetal-arbiter":      "./test_data/controller_config_baremetal_arbiter.yaml",
 		"gcp":                    "./test_data/controller_config_gcp.yaml",
 		"openstack":              "./test_data/controller_config_openstack.yaml",
 		"libvirt":                "./test_data/controller_config_libvirt.yaml",
@@ -281,7 +282,7 @@ func TestGenerateMachineConfigs(t *testing.T) {
 			if err != nil {
 				t.Errorf("Failed to parse Ignition config for %s, %s, error: %v", config, cfg.Name, err)
 			}
-			if role == "master" {
+			if role == masterRole {
 				if !foundPullSecretMaster {
 					foundPullSecretMaster = findIgnFile(ign.Storage.Files, "/var/lib/kubelet/config.json", t)
 				}
@@ -292,7 +293,7 @@ func TestGenerateMachineConfigs(t *testing.T) {
 					foundMTUMigrationMaster = findIgnFile(ign.Storage.Files, "/usr/local/bin/mtu-migration.sh", t)
 					foundMTUMigrationMaster = foundMTUMigrationMaster || findIgnFile(ign.Storage.Files, "/etc/systemd/system/mtu-migration.service", t)
 				}
-			} else if role == "worker" {
+			} else if role == workerRole {
 				if !foundPullSecretWorker {
 					foundPullSecretWorker = findIgnFile(ign.Storage.Files, "/var/lib/kubelet/config.json", t)
 				}
@@ -303,6 +304,18 @@ func TestGenerateMachineConfigs(t *testing.T) {
 					foundMTUMigrationWorker = findIgnFile(ign.Storage.Files, "/usr/local/bin/mtu-migration.sh", t)
 					foundMTUMigrationWorker = foundMTUMigrationWorker || findIgnFile(ign.Storage.Files, "/etc/systemd/system/mtu-migration.service", t)
 				}
+			} else if role == arbiterRole {
+				// arbiter role currently follows master output
+				if !foundPullSecretMaster {
+					foundPullSecretMaster = findIgnFile(ign.Storage.Files, "/var/lib/kubelet/config.json", t)
+				}
+				if !foundKubeletUnitMaster {
+					foundKubeletUnitMaster = findIgnUnit(ign.Systemd.Units, "kubelet.service", t)
+				}
+				if !foundMTUMigrationMaster {
+					foundMTUMigrationMaster = findIgnFile(ign.Storage.Files, "/usr/local/bin/mtu-migration.sh", t)
+					foundMTUMigrationMaster = foundMTUMigrationMaster || findIgnFile(ign.Storage.Files, "/etc/systemd/system/mtu-migration.service", t)
+				}
 			} else {
 				t.Fatalf("Unknown role %s", role)
 			}

pkg/controller/template/test_data/controller_config_baremetal_arbiter.yaml

@@ -0,0 +1,37 @@
+apiVersion: "machineconfigurations.openshift.io/v1"
+kind: "ControllerConfig"
+spec:
+  clusterDNSIP: "10.3.0.10"
+  cloudProviderConfig: ""
+  etcdInitialCount: 3
+  etcdCAData: ZHVtbXkgZXRjZC1jYQo=
+  rootCAData: ZHVtbXkgcm9vdC1jYQo=
+  pullSecret:
+    data: ZHVtbXkgZXRjZC1jYQo=
+  images:
+    etcd: image/etcd:1
+    setupEtcdEnv: image/setupEtcdEnv:1
+    infraImage: image/infraImage:1
+    kubeClientAgentImage: image/kubeClientAgentImage:1
+  infra:
+    apiVersion: config.openshift.io/v1
+    kind: Infrastructure
+    spec:
+      cloudConfig:
+        key: config
+        name: cloud-provider-config
+    status:
+      apiServerInternalURI: https://api-int.my-test-cluster.installer.team.coreos.systems:6443
+      apiServerURL: https://api.my-test-cluster.installer.team.coreos.systems:6443
+      etcdDiscoveryDomain: my-test-cluster.installer.team.coreos.systems
+      infrastructureName: my-test-cluster
+      controlPlaneTopology: HighlyAvailableArbiter
+      platformStatus:
+        type: "BareMetal"
+        baremetal:
+          apiServerInternalIP: 10.0.0.1
+          ingressIP: 10.0.0.2
+          nodeDNSIP: 10.0.0.3
+  dns:
+    spec:
+      baseDomain: my-test-cluster.installer.team.coreos.systems

pkg/daemon/daemon.go

@@ -2728,6 +2728,8 @@ func (dn *Daemon) getControlPlaneTopology() configv1.TopologyMode {
 		return configv1.SingleReplicaTopologyMode
 	case configv1.HighlyAvailableTopologyMode:
 		return configv1.HighlyAvailableTopologyMode
+	case configv1.HighlyAvailableArbiterMode:
+		return configv1.HighlyAvailableArbiterMode
 	default:
 		// for any unhandled case, default to HighlyAvailableTopologyMode
 		return configv1.HighlyAvailableTopologyMode

pkg/operator/bootstrap.go

@@ -155,7 +155,7 @@ func RenderBootstrap(
 		templatectrl.KubeRbacProxyKey:       imgs.KubeRbacProxy,
 	}
 
-	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, nil, []*mcfgv1alpha1.MachineOSConfig{}, nil)
+	config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra, nil, []*mcfgv1alpha1.MachineOSConfig{}, nil)
 
 	manifests := []manifest{
 		{
@@ -182,6 +182,13 @@ func RenderBootstrap(
 		},
 	}
 
+	if infra.Status.ControlPlaneTopology == configv1.HighlyAvailableArbiterMode {
+		manifests = append(manifests, manifest{
+			name:     "manifests/arbiter.machineconfigpool.yaml",
+			filename: "bootstrap/manifests/arbiter.machineconfigpool.yaml",
+		})
+	}
+
 	manifests = appendManifestsByPlatform(manifests, *infra)
 
 	for _, m := range manifests {

pkg/operator/sync.go

@@ -641,9 +641,9 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera
 		}
 
 		// create renderConfig
-		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, pointerConfigData, moscs, apiServer)
+		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra, pointerConfigData, moscs, apiServer)
 	} else {
-		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, pointerConfigData, nil, apiServer)
+		optr.renderConfig = getRenderConfig(optr.namespace, string(kubeAPIServerServingCABytes), spec, &imgs.RenderConfigImages, infra, pointerConfigData, nil, apiServer)
 	}
 
 	return nil
@@ -682,6 +682,11 @@ func (optr *Operator) syncMachineConfigPools(config *renderConfig, _ *configv1.C
 		"manifests/master.machineconfigpool.yaml",
 		"manifests/worker.machineconfigpool.yaml",
 	}
+
+	if config.Infra.Status.ControlPlaneTopology == configv1.HighlyAvailableArbiterMode {
+		mcps = append(mcps, "manifests/arbiter.machineconfigpool.yaml")
+	}
+
 	for _, mcp := range mcps {
 		mcpBytes, err := renderAsset(config, mcp)
 		if err != nil {
@@ -778,6 +783,8 @@ func (optr *Operator) syncMachineConfigNodes(_ *renderConfig, _ *configv1.Cluste
 			pool = "worker"
 		} else if _, ok = node.Labels["node-role.kubernetes.io/master"]; ok {
 			pool = "master"
+		} else if _, ok = node.Labels["node-role.kubernetes.io/arbiter"]; ok {
+			pool = "arbiter"
 		}
 		newMCS := &v1alpha1.MachineConfigNode{
 			Spec: v1alpha1.MachineConfigNodeSpec{
@@ -2035,7 +2042,7 @@ func setGVK(obj runtime.Object, scheme *runtime.Scheme) error {
 	return nil
 }
 
-func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *ctrlcommon.RenderConfigImages, apiServerURL string, pointerConfigData []byte, moscs []*mcfgv1alpha1.MachineOSConfig, apiServer *configv1.APIServer) *renderConfig {
+func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *ctrlcommon.RenderConfigImages, infra *configv1.Infrastructure, pointerConfigData []byte, moscs []*mcfgv1alpha1.MachineOSConfig, apiServer *configv1.APIServer) *renderConfig {
 	tlsMinVersion, tlsCipherSuites := ctrlcommon.GetSecurityProfileCiphersFromAPIServer(apiServer)
 	return &renderConfig{
 		TargetNamespace:        tnamespace,
@@ -2044,8 +2051,9 @@ func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.C
 		ControllerConfig:       *ccSpec,
 		Images:                 imgs,
 		KubeAPIServerServingCA: kubeAPIServerServingCA,
-		APIServerURL:           apiServerURL,
+		APIServerURL:           infra.Status.APIServerInternalURL,
 		PointerConfig:          string(pointerConfigData),
+		Infra:                  *infra,
 		MachineOSConfigs:       moscs,
 		TLSMinVersion:          tlsMinVersion,
 		TLSCipherSuites:        tlsCipherSuites,

pkg/server/bootstrap_server.go

@@ -62,7 +62,7 @@ func NewBootstrapServer(dir, kubeconfig string, ircerts []string) (Server, error
 const yamlExt = ".yaml"
 
 func (bsc *bootstrapServer) GetConfig(cr poolRequest) (*runtime.RawExtension, error) {
-	if cr.machineConfigPool != "master" {
+	if cr.machineConfigPool != "master" && cr.machineConfigPool != "arbiter" {
 		return nil, fmt.Errorf("refusing to serve bootstrap configuration to pool %q", cr.machineConfigPool)
 	}
 	// 1. Read the Machine Config Pool object.

templates/arbiter/00-arbiter/_base/files/kubelet-cgroups.yaml

@@ -0,0 +1,9 @@
+mode: 0644
+path: "/etc/systemd/system.conf.d/kubelet-cgroups.conf"
+contents:
+ inline: |
+    # Turning on Accounting helps track down performance issues.
+    [Manager]
+    DefaultCPUAccounting=yes
+    DefaultMemoryAccounting=yes
+    DefaultBlockIOAccounting=yes

templates/arbiter/00-arbiter/_base/files/usr-local-bin-openshift-kubeconfig-gen.yaml

@@ -0,0 +1,26 @@
+mode: 0755
+path: "/usr/local/bin/recover-kubeconfig.sh"
+contents:
+  inline: |
+    #!/bin/bash
+
+    set -eou pipefail
+
+    # context
+    intapi=$(oc get infrastructures.config.openshift.io cluster -o "jsonpath={.status.apiServerInternalURI}")
+    context="$(oc config current-context)"
+    # cluster
+    cluster="$(oc config view -o "jsonpath={.contexts[?(@.name==\"$context\")].context.cluster}")"
+    server="$(oc config view -o "jsonpath={.clusters[?(@.name==\"$cluster\")].cluster.server}")"
+    # token
+    ca_crt_data="$(oc get secret -n openshift-machine-config-operator node-bootstrapper-token -o "jsonpath={.data.ca\.crt}" | base64 --decode)"
+    namespace="$(oc get secret -n openshift-machine-config-operator node-bootstrapper-token  -o "jsonpath={.data.namespace}" | base64 --decode)"
+    token="$(oc get secret -n openshift-machine-config-operator node-bootstrapper-token -o "jsonpath={.data.token}" | base64 --decode)"
+
+    export KUBECONFIG="$(mktemp)"
+    kubectl config set-credentials "kubelet" --token="$token" >/dev/null
+    ca_crt="$(mktemp)"; echo "$ca_crt_data" > $ca_crt
+    kubectl config set-cluster $cluster --server="$intapi" --certificate-authority="$ca_crt" --embed-certs >/dev/null
+    kubectl config set-context kubelet --cluster="$cluster" --user="kubelet" >/dev/null
+    kubectl config use-context kubelet >/dev/null
+    cat "$KUBECONFIG"

templates/arbiter/00-arbiter/_base/units/rpm-ostreed.service.yaml

@@ -0,0 +1,9 @@
+name: rpm-ostreed.service
+dropins:
+- name: mco-controlplane-nice.conf
+  contents: |
+    # See https://github.com/openshift/machine-config-operator/issues/1897
+    [Service]
+    Nice=10
+    IOSchedulingClass=best-effort
+    IOSchedulingPriority=6

templates/arbiter/00-arbiter/azure/files/etc-kubernetes-manifests-apiserver-watcher.yaml

@@ -0,0 +1,45 @@
+mode: 0644
+path: "/etc/kubernetes/manifests/apiserver-watcher.yaml"
+contents:
+  inline: |
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: apiserver-watcher
+      namespace: openshift-kube-apiserver
+    spec:
+      containers:
+      - name: apiserver-watcher
+        image: "{{.Images.apiServerWatcherKey}}"
+        command:
+          - flock
+          - --verbose
+          - --exclusive
+          - --timeout=300
+          - /rootfs/run/cloud-routes/apiserver-watcher.lock
+          - apiserver-watcher
+        args:
+        - "run"
+        - "--health-check-url={{.Infra.Status.APIServerInternalURL}}/readyz"
+        resources:
+          requests:
+            cpu: 20m
+            memory: 50Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /rootfs
+          name: rootfs
+          mountPropagation: HostToContainer
+      hostNetwork: true
+      hostPID: true
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: "Exists"
+      restartPolicy: Always
+      volumes:
+      - name: rootfs
+        hostPath:
+          path: /
+