Introduce a cluster UUID persisted to a ConfigMap (antrea-io#1805)

Antrea Controller persists a randomly-generated cluster UUID to the antrea-cluster-identity ConfigMap during initialization. The cluster UUID stays the same across all Antrea Controller restarts and Antrea upgrades. The UUID can be consumed by other Antrea components when needed. One current example is the Flow Aggregator, which can use the cluster UUID to generate a deterministic IPFIX Observation Domain ID to uniquely identify the exporting process to the external flow collector. An error during the generation of the cluster UUID will not cause Controller initialization to fail. Instead the Controller will keep retrying until it can successfully 1) establish that a cluster UUID already exists and is correct, or 2) generate and persist a new cluster UUID. If a component strictly depends on the cluster UUID, it should fail its own initialization process until it becomes available.
tnqn · Mar 16, 2021 · 5cc2742 · 5cc2742
1 parent 9efbb30
commit 5cc2742
Show file tree

Hide file tree

Showing 24 changed files with 627 additions and 45 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1093,6 +1093,22 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity-reader
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - antrea-cluster-identity
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   labels:
     app: antrea
@@ -1159,6 +1175,7 @@ rules:
   - ""
   resourceNames:
   - antrea-ca
+  - antrea-cluster-identity
   resources:
   - configmaps
   verbs:
@@ -1299,6 +1316,14 @@ metadata:
   namespace: kube-system
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity
+  namespace: kube-system
+---
+apiVersion: v1
 data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1093,6 +1093,22 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity-reader
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - antrea-cluster-identity
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   labels:
     app: antrea
@@ -1159,6 +1175,7 @@ rules:
   - ""
   resourceNames:
   - antrea-ca
+  - antrea-cluster-identity
   resources:
   - configmaps
   verbs:
@@ -1299,6 +1316,14 @@ metadata:
   namespace: kube-system
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity
+  namespace: kube-system
+---
+apiVersion: v1
 data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -1093,6 +1093,22 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity-reader
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - antrea-cluster-identity
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   labels:
     app: antrea
@@ -1159,6 +1175,7 @@ rules:
   - ""
   resourceNames:
   - antrea-ca
+  - antrea-cluster-identity
   resources:
   - configmaps
   verbs:
@@ -1299,6 +1316,14 @@ metadata:
   namespace: kube-system
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity
+  namespace: kube-system
+---
+apiVersion: v1
 data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -1093,6 +1093,22 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity-reader
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - antrea-cluster-identity
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   labels:
     app: antrea
@@ -1159,6 +1175,7 @@ rules:
   - ""
   resourceNames:
   - antrea-ca
+  - antrea-cluster-identity
   resources:
   - configmaps
   verbs:
@@ -1299,6 +1316,14 @@ metadata:
   namespace: kube-system
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity
+  namespace: kube-system
+---
+apiVersion: v1
 data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -1093,6 +1093,22 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity-reader
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - antrea-cluster-identity
+  resources:
+  - configmaps
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   labels:
     app: antrea
@@ -1159,6 +1175,7 @@ rules:
   - ""
   resourceNames:
   - antrea-ca
+  - antrea-cluster-identity
   resources:
   - configmaps
   verbs:
@@ -1299,6 +1316,14 @@ metadata:
   namespace: kube-system
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-cluster-identity
+  namespace: kube-system
+---
+apiVersion: v1
 data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.

diff --git a/build/yamls/base/cluster-identity-reader.yml b/build/yamls/base/cluster-identity-reader.yml
@@ -0,0 +1,13 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: antrea-cluster-identity-reader
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      - antrea-cluster-identity
+    verbs:
+      - get
diff --git a/build/yamls/base/controller-rbac.yml b/build/yamls/base/controller-rbac.yml
@@ -79,6 +79,7 @@ rules:
       - configmaps
     resourceNames:
       - antrea-ca
+      - antrea-cluster-identity
     verbs:
       - get
       - update

diff --git a/build/yamls/base/controller.yml b/build/yamls/base/controller.yml
@@ -16,6 +16,11 @@ kind: ConfigMap
 metadata:
   name: antrea-ca
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: antrea-cluster-identity
+---
 apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:

diff --git a/build/yamls/base/kustomization.yml b/build/yamls/base/kustomization.yml
@@ -6,6 +6,7 @@ resources:
 - controller.yml
 - agent-rbac.yml
 - agent.yml
+- cluster-identity-reader.yml
 configMapGenerator:
 - files:
   - conf/antrea-controller.conf

diff --git a/build/yamls/flow-aggregator.yml b/build/yamls/flow-aggregator.yml
@@ -103,6 +103,21 @@ subjects:
   name: antrea-agent
   namespace: kube-system
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: flow-aggregator
+  name: flow-aggregator-cluster-id-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: antrea-cluster-identity-reader
+subjects:
+- kind: ServiceAccount
+  name: flow-aggregator
+  namespace: flow-aggregator
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -127,14 +142,22 @@ data:
     # Provide the transport protocol for the flow aggregator collecting process, which is tls, tcp or udp.
     #aggregatorTransportProtocol: "tls"
 
-    # Provide DNS name or IP address of flow aggregator for generating TLS certificate. It must match the flowCollectorAddr parameter in the antrea-agent config.
+    # Provide DNS name or IP address of flow aggregator for generating TLS certificate. It must match
+    # the flowCollectorAddr parameter in the antrea-agent config.
     #flowAggregatorAddress: "flow-aggregator.flow-aggregator.svc"
+
+    # Provide the 32-bit Observation Domain ID which will uniquely identify this instance of the flow
+    # aggregator to an external flow collector. If omitted, an Observation Domain ID will be generated
+    # from the persistent cluster UUID generated by Antrea. Failing that (e.g. because the cluster UUID
+    # is not available), a value will be randomly generated, which may vary across restarts of the flow
+    # aggregator.
+    #observationDomainID:
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: flow-aggregator
-  name: flow-aggregator-configmap-ccfdtmg954
+  name: flow-aggregator-configmap-hf78268hm6
   namespace: flow-aggregator
 ---
 apiVersion: v1
@@ -199,5 +222,5 @@ spec:
       serviceAccountName: flow-aggregator
       volumes:
       - configMap:
-          name: flow-aggregator-configmap-ccfdtmg954
+          name: flow-aggregator-configmap-hf78268hm6
         name: flow-aggregator-config
diff --git a/build/yamls/flow-aggregator/base/conf/flow-aggregator.conf b/build/yamls/flow-aggregator/base/conf/flow-aggregator.conf
@@ -11,5 +11,13 @@
 # Provide the transport protocol for the flow aggregator collecting process, which is tls, tcp or udp.
 #aggregatorTransportProtocol: "tls"
 
-# Provide DNS name or IP address of flow aggregator for generating TLS certificate. It must match the flowCollectorAddr parameter in the antrea-agent config.
+# Provide DNS name or IP address of flow aggregator for generating TLS certificate. It must match
+# the flowCollectorAddr parameter in the antrea-agent config.
 #flowAggregatorAddress: "flow-aggregator.flow-aggregator.svc"
+
+# Provide the 32-bit Observation Domain ID which will uniquely identify this instance of the flow
+# aggregator to an external flow collector. If omitted, an Observation Domain ID will be generated
+# from the persistent cluster UUID generated by Antrea. Failing that (e.g. because the cluster UUID
+# is not available), a value will be randomly generated, which may vary across restarts of the flow
+# aggregator.
+#observationDomainID:
diff --git a/build/yamls/flow-aggregator/base/flow-aggregator.yml b/build/yamls/flow-aggregator/base/flow-aggregator.yml
@@ -77,6 +77,19 @@ roleRef:
   name: flow-exporter-role
   apiGroup: rbac.authorization.k8s.io
 ---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: flow-aggregator-cluster-id-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: antrea-cluster-identity-reader
+subjects:
+  - kind: ServiceAccount
+    name: flow-aggregator
+    namespace: flow-aggregator
+---
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go
@@ -35,6 +35,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/apiserver/openapi"
 	"github.com/vmware-tanzu/antrea/pkg/apiserver/storage"
 	crdinformers "github.com/vmware-tanzu/antrea/pkg/client/informers/externalversions"
+	"github.com/vmware-tanzu/antrea/pkg/clusteridentity"
 	"github.com/vmware-tanzu/antrea/pkg/controller/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy/store"
@@ -47,6 +48,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/monitor"
 	"github.com/vmware-tanzu/antrea/pkg/signals"
 	"github.com/vmware-tanzu/antrea/pkg/util/cipher"
+	"github.com/vmware-tanzu/antrea/pkg/util/env"
 	"github.com/vmware-tanzu/antrea/pkg/version"
 )
 
@@ -106,6 +108,12 @@ func run(o *Options) error {
 	traceflowInformer := crdInformerFactory.Ops().V1alpha1().Traceflows()
 	cgInformer := crdInformerFactory.Core().V1alpha2().ClusterGroups()
 
+	clusterIdentityAllocator := clusteridentity.NewClusterIdentityAllocator(
+		env.GetAntreaNamespace(),
+		clusteridentity.DefaultClusterIdentityConfigMapName,
+		client,
+	)
+
 	// Create Antrea object storage.
 	addressGroupStore := store.NewAddressGroupStore()
 	appliedToGroupStore := store.NewAppliedToGroupStore()
@@ -196,6 +204,8 @@ func run(o *Options) error {
 	informerFactory.Start(stopCh)
 	crdInformerFactory.Start(stopCh)
 
+	go clusterIdentityAllocator.Run(stopCh)
+
 	go controllerMonitor.Run(stopCh)
 
 	go networkPolicyController.Run(stopCh)