From fcda658b8d30dc44515a6b37524b2c24710f3367 Mon Sep 17 00:00:00 2001 From: Dyanngg Date: Fri, 18 Nov 2022 01:13:32 -0800 Subject: [PATCH] Add ClusterSet scope to Antrea policy CRD and LabelIdentityIndex (#3913) Signed-off-by: Yang Ding --- .../charts/antrea/conf/antrea-controller.conf | 10 + .../antrea/crds/clusternetworkpolicy.yaml | 3 + build/charts/antrea/crds/externalnode.yaml | 4 +- build/charts/antrea/crds/networkpolicy.yaml | 3 + .../templates/controller/clusterrole.yaml | 8 + build/charts/antrea/values.yaml | 2 +- build/yamls/antrea-aks.yml | 31 +- build/yamls/antrea-crds.yml | 11 +- build/yamls/antrea-eks.yml | 31 +- build/yamls/antrea-gke.yml | 31 +- build/yamls/antrea-ipsec.yml | 31 +- build/yamls/antrea.yml | 31 +- cmd/antrea-controller/controller.go | 21 +- .../antrea-multicluster-leader-global.yml | 56 ++ ...cluster.crd.antrea.io_resourceexports.yaml | 28 + ...cluster.crd.antrea.io_resourceimports.yaml | 28 + pkg/apis/controlplane/types.go | 3 + pkg/apis/controlplane/v1beta2/generated.pb.go | 420 ++++++++------ pkg/apis/controlplane/v1beta2/generated.proto | 4 + pkg/apis/controlplane/v1beta2/types.go | 3 + .../v1beta2/zz_generated.conversion.go | 2 + .../v1beta2/zz_generated.deepcopy.go | 5 + .../controlplane/zz_generated.deepcopy.go | 5 + pkg/apis/crd/v1alpha1/types.go | 12 + .../handlers/featuregates/handler.go | 2 +- .../handlers/featuregates/handler_test.go | 1 + pkg/apiserver/openapi/zz_generated.openapi.go | 15 + pkg/config/controller/config.go | 11 + pkg/controller/labelidentity/controller.go | 121 ++++ .../labelidentity/controller_test.go | 84 +++ .../labelidentity/label_group_index.go | 508 ++++++++++++++++ .../labelidentity/label_group_index_test.go | 547 ++++++++++++++++++ pkg/controller/networkpolicy/crd_utils.go | 17 +- .../networkpolicy/crd_utils_test.go | 27 + .../networkpolicy/networkpolicy_controller.go | 30 +- .../networkpolicy_controller_test.go | 16 +- 36 files changed, 1960 insertions(+), 202 deletions(-) create mode 100644 pkg/controller/labelidentity/controller.go create mode 100644 pkg/controller/labelidentity/controller_test.go create mode 100644 pkg/controller/labelidentity/label_group_index.go create mode 100644 pkg/controller/labelidentity/label_group_index_test.go diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf index 0cbb0dc0958..fbb62e20429 100644 --- a/build/charts/antrea/conf/antrea-controller.conf +++ b/build/charts/antrea/conf/antrea-controller.conf @@ -43,6 +43,9 @@ featureGates: # Enable collecting support bundle files with SupportBundleCollection CRD. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "SupportBundleCollection" "default" false) }} +# Enable multi-cluster features. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Multicluster" "default" false) }} + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -105,3 +108,10 @@ ipsecCSRSigner: # tls.key: selfSignedCA: {{ .csrSigner.selfSignedCA }} {{- end }} + +multicluster: +{{- with .Values.multicluster }} + # Enable Multicluster which allow Antrea-native policies to select peers + # from other clusters in a ClusterSet. + enable: {{ .enable }} +{{- end }} diff --git a/build/charts/antrea/crds/clusternetworkpolicy.yaml b/build/charts/antrea/crds/clusternetworkpolicy.yaml index 417e827cbf3..1e274e61004 100644 --- a/build/charts/antrea/crds/clusternetworkpolicy.yaml +++ b/build/charts/antrea/crds/clusternetworkpolicy.yaml @@ -359,6 +359,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: diff --git a/build/charts/antrea/crds/externalnode.yaml b/build/charts/antrea/crds/externalnode.yaml index bd310f003ae..8584dd010f4 100644 --- a/build/charts/antrea/crds/externalnode.yaml +++ b/build/charts/antrea/crds/externalnode.yaml @@ -42,8 +42,8 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode \ No newline at end of file diff --git a/build/charts/antrea/crds/networkpolicy.yaml b/build/charts/antrea/crds/networkpolicy.yaml index 88f2fdbeb58..ba6d337a0a7 100644 --- a/build/charts/antrea/crds/networkpolicy.yaml +++ b/build/charts/antrea/crds/networkpolicy.yaml @@ -302,6 +302,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: diff --git a/build/charts/antrea/templates/controller/clusterrole.yaml b/build/charts/antrea/templates/controller/clusterrole.yaml index 40b4d2c903e..8db2c19a53f 100644 --- a/build/charts/antrea/templates/controller/clusterrole.yaml +++ b/build/charts/antrea/templates/controller/clusterrole.yaml @@ -304,3 +304,11 @@ rules: - supportbundlecollections/status verbs: - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index a328b546bfc..9dbdcb0f1e4 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -299,7 +299,7 @@ logVerbosity: 0 whereabouts: enable: false -## -- Configure Multicluster, for use by the antrea-agent. +## -- Configure Multicluster defaults for both Antrea controller and agent. multicluster: # -- Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. # This feature is supported only with encap mode. diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 7270cb8126c..21dc0e147c7 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -737,6 +737,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -1347,11 +1350,12 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode + --- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 @@ -1792,6 +1796,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -3211,6 +3218,9 @@ data: # Enable collecting support bundle files with SupportBundleCollection CRD. # SupportBundleCollection: false + # Enable multi-cluster features. + # Multicluster: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3266,6 +3276,11 @@ data: # tls.crt: # tls.key: selfSignedCA: true + + multicluster: + # Enable Multicluster which allow Antrea-native policies to select peers + # from other clusters in a ClusterSet. + enable: false --- # Source: antrea/templates/crds/group.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3988,6 +4003,14 @@ rules: - supportbundlecollections/status verbs: - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch --- # Source: antrea/templates/crds-rbac/clusterroles.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4165,7 +4188,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 0e08440d83a645b8e7df882d1225808a78acfa9320db5c174620ffdbe6434caa + checksum/config: 4e2311619bbb44c5903fc863c0df9616409367bdc5316eb4c8f677cbab5bad04 labels: app: antrea component: antrea-agent @@ -4406,7 +4429,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 0e08440d83a645b8e7df882d1225808a78acfa9320db5c174620ffdbe6434caa + checksum/config: 4e2311619bbb44c5903fc863c0df9616409367bdc5316eb4c8f677cbab5bad04 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 9f0756926e2..baff751f0a3 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -730,6 +730,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -1332,11 +1335,12 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode--- +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -1773,6 +1777,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index cbcddaab646..3e27f53fd59 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -737,6 +737,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -1347,11 +1350,12 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode + --- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 @@ -1792,6 +1796,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -3211,6 +3218,9 @@ data: # Enable collecting support bundle files with SupportBundleCollection CRD. # SupportBundleCollection: false + # Enable multi-cluster features. + # Multicluster: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3266,6 +3276,11 @@ data: # tls.crt: # tls.key: selfSignedCA: true + + multicluster: + # Enable Multicluster which allow Antrea-native policies to select peers + # from other clusters in a ClusterSet. + enable: false --- # Source: antrea/templates/crds/group.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3988,6 +4003,14 @@ rules: - supportbundlecollections/status verbs: - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch --- # Source: antrea/templates/crds-rbac/clusterroles.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4165,7 +4188,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 0e08440d83a645b8e7df882d1225808a78acfa9320db5c174620ffdbe6434caa + checksum/config: 4e2311619bbb44c5903fc863c0df9616409367bdc5316eb4c8f677cbab5bad04 labels: app: antrea component: antrea-agent @@ -4408,7 +4431,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 0e08440d83a645b8e7df882d1225808a78acfa9320db5c174620ffdbe6434caa + checksum/config: 4e2311619bbb44c5903fc863c0df9616409367bdc5316eb4c8f677cbab5bad04 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index bb5e434e842..18422505f45 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -737,6 +737,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -1347,11 +1350,12 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode + --- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 @@ -1792,6 +1796,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -3211,6 +3218,9 @@ data: # Enable collecting support bundle files with SupportBundleCollection CRD. # SupportBundleCollection: false + # Enable multi-cluster features. + # Multicluster: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3266,6 +3276,11 @@ data: # tls.crt: # tls.key: selfSignedCA: true + + multicluster: + # Enable Multicluster which allow Antrea-native policies to select peers + # from other clusters in a ClusterSet. + enable: false --- # Source: antrea/templates/crds/group.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3988,6 +4003,14 @@ rules: - supportbundlecollections/status verbs: - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch --- # Source: antrea/templates/crds-rbac/clusterroles.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4165,7 +4188,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 4aaa9260916de6b712215a4deb2b7694dceb17dba615f691f61aaecdd6a61abb + checksum/config: 6244d1a441fcf006e951fdced30f7a591bfeb6fcb2eb1277bc1c91304f3cf1c3 labels: app: antrea component: antrea-agent @@ -4405,7 +4428,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 4aaa9260916de6b712215a4deb2b7694dceb17dba615f691f61aaecdd6a61abb + checksum/config: 6244d1a441fcf006e951fdced30f7a591bfeb6fcb2eb1277bc1c91304f3cf1c3 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index d83d7b5cf24..951078527b0 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -737,6 +737,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -1347,11 +1350,12 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode + --- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 @@ -1792,6 +1796,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -3224,6 +3231,9 @@ data: # Enable collecting support bundle files with SupportBundleCollection CRD. # SupportBundleCollection: false + # Enable multi-cluster features. + # Multicluster: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3279,6 +3289,11 @@ data: # tls.crt: # tls.key: selfSignedCA: true + + multicluster: + # Enable Multicluster which allow Antrea-native policies to select peers + # from other clusters in a ClusterSet. + enable: false --- # Source: antrea/templates/crds/group.yaml apiVersion: apiextensions.k8s.io/v1 @@ -4001,6 +4016,14 @@ rules: - supportbundlecollections/status verbs: - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch --- # Source: antrea/templates/crds-rbac/clusterroles.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4178,7 +4201,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 19d1abdf4209cb57d22d60c3c4318bc8f822b4ad028faa1f18521a194db726fd + checksum/config: 3276ac7809f4f3c6ab0d9b9730b46b35cd398d1f9ab7475b8bf47ad757a58f94 checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -4464,7 +4487,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 19d1abdf4209cb57d22d60c3c4318bc8f822b4ad028faa1f18521a194db726fd + checksum/config: 3276ac7809f4f3c6ab0d9b9730b46b35cd398d1f9ab7475b8bf47ad757a58f94 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 415977ed2b6..496a852b57d 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -737,6 +737,9 @@ spec: type: array matchLabels: x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -1347,11 +1350,12 @@ spec: storage: true scope: Namespaced names: - kind: ExternalNode plural: externalnodes + singular: externalnode + kind: ExternalNode shortNames: - en - singular: externalnode + --- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 @@ -1792,6 +1796,9 @@ spec: x-kubernetes-preserve-unknown-fields: true group: type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] name: type: string enableLogging: @@ -3211,6 +3218,9 @@ data: # Enable collecting support bundle files with SupportBundleCollection CRD. # SupportBundleCollection: false + # Enable multi-cluster features. + # Multicluster: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3266,6 +3276,11 @@ data: # tls.crt: # tls.key: selfSignedCA: true + + multicluster: + # Enable Multicluster which allow Antrea-native policies to select peers + # from other clusters in a ClusterSet. + enable: false --- # Source: antrea/templates/crds/group.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3988,6 +4003,14 @@ rules: - supportbundlecollections/status verbs: - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch --- # Source: antrea/templates/crds-rbac/clusterroles.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -4165,7 +4188,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 595e4f3c16ca5f4f27b99c1020d5575a2eeab3522d95ed17825f561917764a4d + checksum/config: 9df2527a52bfb6aef5f0b520cde140b5434c77b3da2e85e6e4d2ca713627d1b8 labels: app: antrea component: antrea-agent @@ -4405,7 +4428,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 595e4f3c16ca5f4f27b99c1020d5575a2eeab3522d95ed17825f561917764a4d + checksum/config: 9df2527a52bfb6aef5f0b520cde140b5434c77b3da2e85e6e4d2ca713627d1b8 labels: app: antrea component: antrea-controller diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 9114ea2d2fb..d844024a6a1 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -37,6 +37,7 @@ import ( aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" netutils "k8s.io/utils/net" + mcinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions" antreaapis "antrea.io/antrea/pkg/apis" "antrea.io/antrea/pkg/apiserver" "antrea.io/antrea/pkg/apiserver/certificate" @@ -51,6 +52,7 @@ import ( "antrea.io/antrea/pkg/controller/externalnode" "antrea.io/antrea/pkg/controller/grouping" antreaipam "antrea.io/antrea/pkg/controller/ipam" + "antrea.io/antrea/pkg/controller/labelidentity" "antrea.io/antrea/pkg/controller/metrics" "antrea.io/antrea/pkg/controller/networkpolicy" "antrea.io/antrea/pkg/controller/networkpolicy/store" @@ -116,7 +118,7 @@ func run(o *Options) error { // Create K8s Clientset, Aggregator Clientset, CRD Clientset and SharedInformerFactory for the given config. // Aggregator Clientset is used to update the CABundle of the APIServices backed by antrea-controller so that // the aggregator can verify its serving certificate. - client, aggregatorClient, crdClient, apiExtensionClient, _, err := k8s.CreateClients(o.config.ClientConnection, "") + client, aggregatorClient, crdClient, apiExtensionClient, mcClient, err := k8s.CreateClients(o.config.ClientConnection, "") if err != nil { return fmt.Errorf("error creating K8s clients: %v", err) } @@ -152,10 +154,13 @@ func run(o *Options) error { groupStore := store.NewGroupStore() groupEntityIndex := grouping.NewGroupEntityIndex() groupEntityController := grouping.NewGroupEntityController(groupEntityIndex, podInformer, namespaceInformer, eeInformer) + labelIdentityIndex := labelidentity.NewLabelIdentityIndex() + multiclusterEnabled := features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.Enable networkPolicyController := networkpolicy.NewNetworkPolicyController(client, crdClient, groupEntityIndex, + labelIdentityIndex, namespaceInformer, serviceInformer, networkPolicyInformer, @@ -168,7 +173,8 @@ func run(o *Options) error { addressGroupStore, appliedToGroupStore, networkPolicyStore, - groupStore) + groupStore, + multiclusterEnabled) var externalNodeController *externalnode.ExternalNodeController if features.DefaultFeatureGate.Enabled(features.ExternalNode) { @@ -311,6 +317,17 @@ func run(o *Options) error { go groupEntityController.Run(stopCh) + if multiclusterEnabled { + mcInformerFactoty := mcinformers.NewSharedInformerFactory(mcClient, informerDefaultResync) + labelIdentityInformer := mcInformerFactoty.Multicluster().V1alpha1().LabelIdentities() + labelIdentityController := labelidentity.NewLabelIdentityController(labelIdentityIndex, labelIdentityInformer) + mcInformerFactoty.Start(stopCh) + + go labelIdentityIndex.Run(stopCh) + + go labelIdentityController.Run(stopCh) + } + go networkPolicyController.Run(stopCh) go apiServer.Run(ctx) diff --git a/multicluster/build/yamls/antrea-multicluster-leader-global.yml b/multicluster/build/yamls/antrea-multicluster-leader-global.yml index b9803b10573..17ba41431c9 100644 --- a/multicluster/build/yamls/antrea-multicluster-leader-global.yml +++ b/multicluster/build/yamls/antrea-multicluster-leader-global.yml @@ -634,6 +634,10 @@ spec: contains only "value". The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in either policy @@ -925,6 +929,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1209,6 +1217,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1562,6 +1574,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1875,6 +1891,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -2159,6 +2179,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -2512,6 +2536,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -3534,6 +3562,10 @@ spec: contains only "value". The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in either policy @@ -3825,6 +3857,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -4109,6 +4145,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -4462,6 +4502,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -4775,6 +4819,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -5059,6 +5107,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -5412,6 +5464,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in diff --git a/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceexports.yaml b/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceexports.yaml index a971352d608..3080793636e 100644 --- a/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceexports.yaml +++ b/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceexports.yaml @@ -329,6 +329,10 @@ spec: contains only "value". The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in either policy @@ -620,6 +624,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -904,6 +912,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1257,6 +1269,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1570,6 +1586,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1854,6 +1874,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -2207,6 +2231,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in diff --git a/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceimports.yaml b/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceimports.yaml index d46930500f2..f32b24ccb64 100644 --- a/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceimports.yaml +++ b/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceimports.yaml @@ -327,6 +327,10 @@ spec: contains only "value". The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in either policy @@ -618,6 +622,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -902,6 +910,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1255,6 +1267,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1568,6 +1584,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -1852,6 +1872,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in @@ -2205,6 +2229,10 @@ spec: The requirements are ANDed. type: object type: object + scope: + description: Define scope of the Pod/NamespaceSelector(s) + of this peer. Can only be used in ingress NetworkPolicyPeers. + type: string service: description: Select a certain Service which matches the NamespacedName. A Service can only be set in diff --git a/pkg/apis/controlplane/types.go b/pkg/apis/controlplane/types.go index 2ebfb37fafe..75001db7826 100644 --- a/pkg/apis/controlplane/types.go +++ b/pkg/apis/controlplane/types.go @@ -314,6 +314,9 @@ type NetworkPolicyPeer struct { // A list of ServiceReference. // This field can only be possibly set for NetworkPolicyPeer of egress rules. ToServices []ServiceReference + // A list of labelIdentities selected as ingress peers for stretched policy. + // This field can only be possibly set for NetworkPolicyPeer of ingress rules. + LabelIdentities []uint32 } // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24"). The except entry describes CIDRs that should diff --git a/pkg/apis/controlplane/v1beta2/generated.pb.go b/pkg/apis/controlplane/v1beta2/generated.pb.go index 9535166a5f0..fbf40acf374 100644 --- a/pkg/apis/controlplane/v1beta2/generated.pb.go +++ b/pkg/apis/controlplane/v1beta2/generated.pb.go @@ -1186,171 +1186,172 @@ func init() { } var fileDescriptor_fbaa7d016762fa1d = []byte{ - // 2610 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0xcb, 0x73, 0x1c, 0x47, - 0xf9, 0x9e, 0x7d, 0x48, 0xda, 0x4f, 0xaf, 0x55, 0x2b, 0x89, 0xf5, 0x4b, 0x62, 0xc9, 0x9e, 0xfc, - 0x48, 0x19, 0x2a, 0xec, 0x46, 0xc2, 0x8e, 0x0d, 0x8e, 0x03, 0x5a, 0x59, 0x56, 0x2d, 0xb1, 0xe4, - 0x4d, 0x4b, 0x2e, 0x57, 0x11, 0x1c, 0x32, 0x9a, 0xe9, 0x5d, 0x35, 0x9a, 0x9d, 0x99, 0xf4, 0xf4, - 0x28, 0x16, 0x07, 0x2a, 0x14, 0x70, 0x08, 0x2f, 0x53, 0x5c, 0x28, 0x6e, 0x70, 0xe2, 0xc2, 0x5f, - 0x90, 0x1b, 0x37, 0x17, 0xa7, 0x50, 0x14, 0x45, 0x4e, 0x2a, 0x2c, 0x0a, 0xa8, 0x1c, 0xe0, 0xc0, - 0x0d, 0x73, 0xa1, 0xba, 0xa7, 0xe7, 0xb5, 0x0f, 0xcb, 0x2b, 0xc9, 0xa2, 0x0a, 0x72, 0xd2, 0x4e, - 0x7f, 0xcf, 0xee, 0xef, 0xfb, 0xfa, 0x7b, 0xb4, 0xe0, 0x35, 0xc3, 0xe1, 0x8c, 0x18, 0x15, 0xea, - 0x56, 0xc3, 0x5f, 0x55, 0x6f, 0xbb, 0x55, 0x35, 0x3c, 0xea, 0x57, 0x4d, 0xd7, 0xe1, 0xcc, 0xb5, - 0x3d, 0xdb, 0x70, 0x48, 0x75, 0x67, 0x7e, 0x93, 0x70, 0x63, 0xa1, 0xda, 0x22, 0x0e, 0x61, 0x06, - 0x27, 0x56, 0xc5, 0x63, 0x2e, 0x77, 0x51, 0x25, 0xa4, 0xfa, 0x1a, 0x75, 0xd5, 0xaf, 0x8a, 0xb7, - 0xdd, 0xaa, 0x08, 0xfa, 0x4a, 0x9a, 0xbe, 0xa2, 0xe8, 0x9f, 0xbd, 0xdc, 0x5f, 0x9e, 0xcf, 0x0d, - 0xee, 0x57, 0x77, 0xe6, 0x0d, 0xdb, 0xdb, 0x32, 0xe6, 0x3b, 0x25, 0x3d, 0xfb, 0xd9, 0x16, 0xe5, - 0x5b, 0xc1, 0x66, 0xc5, 0x74, 0xdb, 0xd5, 0x96, 0xdb, 0x72, 0xab, 0x72, 0x79, 0x33, 0x68, 0xca, - 0x2f, 0xf9, 0x21, 0x7f, 0x29, 0xf4, 0x0b, 0xdb, 0x97, 0x7d, 0x29, 0xc5, 0xa3, 0x6d, 0xc3, 0xdc, - 0xa2, 0x0e, 0x61, 0xbb, 0x89, 0xac, 0x36, 0xe1, 0x46, 0x75, 0xa7, 0x5b, 0x48, 0xb5, 0x1f, 0x15, - 0x0b, 0x1c, 0x4e, 0xdb, 0xa4, 0x8b, 0xe0, 0x95, 0x83, 0x08, 0x7c, 0x73, 0x8b, 0xb4, 0x8d, 0x2e, - 0xba, 0xcf, 0xf5, 0xa3, 0x0b, 0x38, 0xb5, 0xab, 0xd4, 0xe1, 0x3e, 0x67, 0x9d, 0x44, 0xfa, 0x5f, - 0x35, 0x18, 0x5b, 0xb4, 0x2c, 0x46, 0x7c, 0x7f, 0x85, 0xb9, 0x81, 0x87, 0xde, 0x86, 0x11, 0xb1, - 0x13, 0xcb, 0xe0, 0xc6, 0x8c, 0x76, 0x56, 0x3b, 0x3f, 0xba, 0xf0, 0x72, 0x25, 0x64, 0x5c, 0x49, - 0x33, 0x4e, 0x6c, 0x22, 0xb0, 0x2b, 0x3b, 0xf3, 0x95, 0x9b, 0x9b, 0x5f, 0x27, 0x26, 0x5f, 0x25, - 0xdc, 0xa8, 0xa1, 0xfb, 0x7b, 0x73, 0xa7, 0xf6, 0xf7, 0xe6, 0x20, 0x59, 0xc3, 0x31, 0x57, 0x14, - 0xc0, 0x58, 0x4b, 0x88, 0x5a, 0x25, 0xed, 0x4d, 0xc2, 0xfc, 0x99, 0xdc, 0xd9, 0xfc, 0xf9, 0xd1, - 0x85, 0x2b, 0x03, 0x9a, 0xbd, 0xb2, 0x92, 0xf0, 0xa8, 0x3d, 0xa5, 0x04, 0x8e, 0xa5, 0x16, 0x7d, - 0x9c, 0x11, 0xa3, 0xff, 0x4e, 0x83, 0x72, 0x7a, 0xa7, 0x37, 0xa8, 0xcf, 0xd1, 0x57, 0xbb, 0x76, - 0x5b, 0x79, 0xbc, 0xdd, 0x0a, 0x6a, 0xb9, 0xd7, 0xb2, 0x12, 0x3d, 0x12, 0xad, 0xa4, 0x76, 0x6a, - 0x40, 0x91, 0x72, 0xd2, 0x8e, 0xb6, 0xf8, 0xea, 0xa0, 0x5b, 0x4c, 0xab, 0x5b, 0x1b, 0x57, 0x82, - 0x8a, 0x75, 0xc1, 0x12, 0x87, 0x9c, 0xf5, 0xf7, 0xf3, 0x30, 0x95, 0x46, 0x6b, 0x18, 0xdc, 0xdc, - 0x3a, 0x01, 0x23, 0x7e, 0x47, 0x83, 0x29, 0xc3, 0xb2, 0x88, 0xb5, 0x72, 0xcc, 0xa6, 0xfc, 0x3f, - 0x25, 0x56, 0xec, 0x2a, 0xcb, 0x1d, 0x77, 0x0b, 0x44, 0xdf, 0xd3, 0x60, 0x9a, 0x91, 0xb6, 0xbb, - 0xd3, 0xa1, 0x48, 0xfe, 0xe8, 0x8a, 0x3c, 0xa7, 0x14, 0x99, 0xc6, 0xdd, 0xfc, 0x71, 0x2f, 0xa1, - 0xfa, 0xc7, 0x1a, 0x4c, 0x2c, 0x7a, 0x9e, 0x4d, 0x89, 0xb5, 0xe1, 0xfe, 0x97, 0x47, 0xd3, 0x1f, - 0x34, 0x40, 0xd9, 0xbd, 0x9e, 0x40, 0x3c, 0x99, 0xd9, 0x78, 0x7a, 0x6d, 0xe0, 0x78, 0xca, 0x28, - 0xdc, 0x27, 0xa2, 0xbe, 0x9f, 0x87, 0xe9, 0x2c, 0xe2, 0x27, 0x31, 0xf5, 0x9f, 0x8b, 0xa9, 0x77, - 0x60, 0xba, 0x66, 0xf8, 0xd4, 0x5c, 0x0c, 0xf8, 0x16, 0x71, 0x38, 0x35, 0x0d, 0x4e, 0x5d, 0x07, - 0xbd, 0x04, 0x23, 0x81, 0x4f, 0x98, 0x63, 0xb4, 0x89, 0x34, 0x46, 0x29, 0xf1, 0x9b, 0x5b, 0x6a, - 0x1d, 0xc7, 0x18, 0x02, 0xdb, 0x33, 0x7c, 0xff, 0x5d, 0x97, 0x59, 0x33, 0xb9, 0x2c, 0x76, 0x43, - 0xad, 0xe3, 0x18, 0x43, 0x9f, 0x87, 0x72, 0x2d, 0x70, 0x2c, 0x9b, 0x5c, 0xa7, 0x36, 0x59, 0x27, - 0x6c, 0x87, 0x30, 0x74, 0x06, 0xf2, 0x01, 0xb3, 0x95, 0xa8, 0x51, 0x45, 0x9c, 0xbf, 0x85, 0x6f, - 0x60, 0xb1, 0xae, 0xdf, 0xcb, 0xc1, 0x99, 0x90, 0x26, 0xc4, 0x17, 0xda, 0x2e, 0xb9, 0x4e, 0x93, - 0xb6, 0x02, 0x16, 0x2a, 0x7c, 0x11, 0x46, 0x37, 0x89, 0xc1, 0x08, 0xdb, 0x70, 0xb7, 0x89, 0xa3, - 0x18, 0x4d, 0x2b, 0x46, 0xa3, 0xb5, 0x04, 0x84, 0xd3, 0x78, 0xe8, 0x45, 0x18, 0x32, 0x3c, 0xfa, - 0x3a, 0xd9, 0x55, 0x7a, 0x4f, 0x28, 0x8a, 0xa1, 0xc5, 0x46, 0xfd, 0x75, 0xb2, 0x8b, 0x15, 0x14, - 0xfd, 0x48, 0x83, 0xe9, 0xcd, 0xee, 0x73, 0x9a, 0xc9, 0x4b, 0x47, 0x5d, 0x1a, 0xd4, 0x66, 0x3d, - 0x8e, 0xbc, 0x76, 0x5a, 0xd8, 0xad, 0x07, 0x00, 0xf7, 0x12, 0xac, 0xff, 0xbc, 0x00, 0xd3, 0x4b, - 0x76, 0xe0, 0x73, 0xc2, 0x32, 0xce, 0xf5, 0xe4, 0xa3, 0xe8, 0x5b, 0x1a, 0x94, 0x49, 0xb3, 0x49, - 0x4c, 0x4e, 0x77, 0xc8, 0x31, 0x06, 0xd1, 0x8c, 0x92, 0x5a, 0x5e, 0xee, 0x60, 0x8e, 0xbb, 0xc4, - 0xa1, 0x6f, 0xc2, 0x54, 0xbc, 0x56, 0x6f, 0xd4, 0x6c, 0xd7, 0xdc, 0x8e, 0xe2, 0xe7, 0xe2, 0xa0, - 0x3a, 0xd4, 0x1b, 0x6b, 0x84, 0x27, 0x21, 0xbc, 0xdc, 0xc9, 0x17, 0x77, 0x8b, 0x42, 0x97, 0x61, - 0x8c, 0xbb, 0xdc, 0xb0, 0xa3, 0xed, 0x17, 0xce, 0x6a, 0xe7, 0xf3, 0xc9, 0xbd, 0xbe, 0x91, 0x82, - 0xe1, 0x0c, 0x26, 0x5a, 0x00, 0x90, 0xdf, 0x0d, 0xa3, 0x45, 0xfc, 0x99, 0xa2, 0xa4, 0x8b, 0xcf, - 0x7b, 0x23, 0x86, 0xe0, 0x14, 0x96, 0xf0, 0x6d, 0x33, 0x60, 0x8c, 0x38, 0x5c, 0x7c, 0xcf, 0x0c, - 0x49, 0xa2, 0xd8, 0xb7, 0x97, 0x12, 0x10, 0x4e, 0xe3, 0xe9, 0x7f, 0xd1, 0x60, 0x74, 0xb9, 0xf5, - 0x3f, 0x50, 0x79, 0xfe, 0x56, 0x83, 0xc9, 0xd4, 0x46, 0x4f, 0x20, 0x51, 0xbe, 0x9d, 0x4d, 0x94, - 0x03, 0xef, 0x30, 0xa5, 0x6d, 0x9f, 0x2c, 0xf9, 0x83, 0x3c, 0x94, 0x53, 0x58, 0x61, 0x8a, 0xb4, - 0x00, 0xdc, 0xf8, 0xdc, 0x8f, 0xd5, 0x86, 0x29, 0xbe, 0x9f, 0xa4, 0xc9, 0x1e, 0x69, 0xd2, 0x86, - 0xd3, 0xcb, 0x77, 0xb9, 0x48, 0x77, 0xf6, 0xb2, 0xc3, 0x29, 0xdf, 0xc5, 0xa4, 0x49, 0x18, 0x71, - 0x4c, 0x82, 0xce, 0x42, 0x21, 0x95, 0x26, 0xc7, 0x14, 0xeb, 0xc2, 0x9a, 0x48, 0x91, 0x12, 0x82, - 0xaa, 0x50, 0x12, 0x7f, 0x7d, 0xcf, 0x30, 0x89, 0xca, 0x33, 0x53, 0x0a, 0xad, 0xb4, 0x16, 0x01, - 0x70, 0x82, 0xa3, 0xff, 0x4b, 0x83, 0xb2, 0x14, 0xbf, 0xe8, 0xfb, 0xae, 0x49, 0xc3, 0x0c, 0x77, - 0x22, 0xf5, 0x51, 0xd9, 0x50, 0x12, 0xd5, 0xfe, 0x0f, 0x5d, 0x0a, 0x4a, 0xea, 0xf8, 0x90, 0x92, - 0xcb, 0x7d, 0xb1, 0x83, 0x3f, 0xee, 0x92, 0xa8, 0x7f, 0x50, 0x80, 0xd1, 0xd4, 0xe1, 0xa3, 0xdb, - 0x90, 0xf7, 0x5c, 0x4b, 0xed, 0x79, 0xe0, 0x1e, 0xaf, 0xe1, 0x5a, 0x89, 0x1a, 0xc3, 0xa2, 0xaa, - 0x10, 0x2b, 0x82, 0x23, 0xfa, 0xb6, 0x06, 0x13, 0x24, 0x63, 0x55, 0x69, 0x9d, 0xd1, 0x85, 0x95, - 0x81, 0xe3, 0xb9, 0xb7, 0x6f, 0xd4, 0xd0, 0xfe, 0xde, 0xdc, 0x44, 0x07, 0xb0, 0x43, 0x24, 0x7a, - 0x11, 0xf2, 0xd4, 0x0b, 0xdd, 0x7a, 0xac, 0xf6, 0x94, 0x50, 0xb0, 0xde, 0xf0, 0x1f, 0xee, 0xcd, - 0x95, 0xea, 0x0d, 0xd5, 0x78, 0x62, 0x81, 0x80, 0xde, 0x82, 0xa2, 0xe7, 0x32, 0x2e, 0x92, 0x8d, - 0xb0, 0xc8, 0xe7, 0x07, 0xd5, 0x51, 0x78, 0x9a, 0xd5, 0x70, 0x19, 0x4f, 0x6e, 0x1c, 0xf1, 0xe5, - 0xe3, 0x90, 0x2d, 0x7a, 0x13, 0x0a, 0x8e, 0x6b, 0x11, 0x99, 0x93, 0x46, 0x17, 0xae, 0x0e, 0xcc, - 0xde, 0xb5, 0x48, 0xb2, 0xf1, 0x11, 0x19, 0x02, 0x62, 0x49, 0x32, 0x45, 0x2d, 0x18, 0xf6, 0x09, - 0xdb, 0xa1, 0x66, 0x98, 0xbe, 0x46, 0x17, 0xbe, 0x34, 0x28, 0xff, 0xf5, 0x90, 0x3c, 0x11, 0x31, - 0xba, 0xbf, 0x37, 0x37, 0x1c, 0xad, 0x46, 0xdc, 0xf5, 0x5f, 0x6a, 0x30, 0x91, 0xf5, 0xbd, 0x6c, - 0xf8, 0x69, 0x07, 0x87, 0x5f, 0x1c, 0xd1, 0xb9, 0xbe, 0x11, 0x5d, 0x83, 0x7c, 0x40, 0x2d, 0x59, - 0xfd, 0x95, 0x6a, 0x2f, 0xc7, 0xe5, 0x6a, 0xfd, 0xda, 0xc3, 0xbd, 0xb9, 0x73, 0xfd, 0xc6, 0x44, - 0x7c, 0xd7, 0x23, 0x7e, 0xe5, 0x56, 0xfd, 0x1a, 0x16, 0xc4, 0xfa, 0xaf, 0x35, 0x18, 0x56, 0x05, - 0x05, 0xba, 0x0d, 0x05, 0x93, 0x5a, 0x4c, 0xf9, 0xf8, 0x21, 0x4b, 0x98, 0x58, 0xd1, 0xa5, 0xfa, - 0x35, 0x8c, 0x25, 0x43, 0x74, 0x07, 0x86, 0xc8, 0x5d, 0x93, 0x78, 0x5c, 0xc5, 0xf1, 0x21, 0x59, - 0xc7, 0x65, 0xf1, 0xb2, 0x64, 0x86, 0x15, 0x53, 0xbd, 0x09, 0x45, 0x89, 0x80, 0x5e, 0x80, 0x1c, - 0xf5, 0xa4, 0xfa, 0x63, 0xb5, 0xe9, 0xfd, 0xbd, 0xb9, 0x5c, 0xbd, 0x91, 0x75, 0xe1, 0x1c, 0xf5, - 0x44, 0xd5, 0xe4, 0x31, 0xd2, 0xa4, 0x77, 0x6f, 0x10, 0xa7, 0xc5, 0xb7, 0xe4, 0xf9, 0x16, 0x93, - 0x0c, 0xdf, 0x48, 0xc1, 0x70, 0x06, 0x53, 0xff, 0x99, 0x06, 0x68, 0x35, 0xb0, 0x45, 0xf5, 0xeb, - 0x73, 0x69, 0xde, 0xba, 0xd3, 0x74, 0xd1, 0x0b, 0x50, 0x94, 0x85, 0x80, 0xb2, 0x6a, 0xec, 0xd7, - 0xa1, 0x03, 0x84, 0x30, 0xf4, 0x16, 0x14, 0x3c, 0xd7, 0x3a, 0xf4, 0x8c, 0x28, 0x73, 0x7f, 0xc4, - 0x47, 0xdc, 0x70, 0x2d, 0x1f, 0x4b, 0xbe, 0xfa, 0xfb, 0x1a, 0x94, 0xe2, 0xd8, 0x12, 0xbe, 0x23, - 0xc2, 0x49, 0x6a, 0x54, 0x4c, 0xe3, 0x33, 0x8e, 0x25, 0xe4, 0x31, 0xbc, 0xeb, 0x32, 0x8c, 0xc8, - 0xe1, 0xa1, 0xe9, 0xda, 0xca, 0xc5, 0x9e, 0x8f, 0xdb, 0x29, 0xb5, 0xfe, 0x30, 0xf5, 0x1b, 0xc7, - 0xd8, 0xfa, 0xdf, 0xf2, 0x30, 0xbe, 0x46, 0xf8, 0xbb, 0x2e, 0xdb, 0x6e, 0xb8, 0x36, 0x35, 0x77, - 0x4f, 0x20, 0x6b, 0x34, 0xa1, 0xc8, 0x02, 0x9b, 0x44, 0x07, 0xbc, 0x38, 0xf0, 0xc5, 0x91, 0xd6, - 0x17, 0x07, 0x36, 0x49, 0xec, 0x28, 0xbe, 0x7c, 0x1c, 0xb2, 0x47, 0x57, 0x61, 0xd2, 0xc8, 0x8c, - 0x0d, 0xc2, 0x3b, 0xb3, 0x24, 0xfd, 0x6d, 0x32, 0x3b, 0x51, 0xf0, 0x71, 0x27, 0x2e, 0x3a, 0x2f, - 0x0e, 0x95, 0xba, 0x4c, 0xdc, 0xf2, 0xa2, 0x5c, 0xd7, 0x6a, 0x63, 0xe1, 0x81, 0x86, 0x6b, 0x38, - 0x86, 0xa2, 0x0b, 0x30, 0xc6, 0x29, 0x61, 0x11, 0x44, 0x5e, 0x88, 0xc5, 0x5a, 0x59, 0x16, 0xf6, - 0xa9, 0x75, 0x9c, 0xc1, 0x42, 0x3e, 0x94, 0x7c, 0x37, 0x60, 0xf2, 0x86, 0x52, 0x77, 0xdc, 0xf5, - 0xa3, 0x1d, 0x45, 0xec, 0x75, 0xe3, 0xe2, 0xa6, 0x5a, 0x8f, 0x98, 0xe3, 0x44, 0x8e, 0xfe, 0x7b, - 0x0d, 0xa6, 0x32, 0x44, 0x27, 0x50, 0xfb, 0x6e, 0x66, 0x6b, 0xdf, 0xab, 0x47, 0xda, 0x64, 0x9f, - 0xea, 0xf7, 0x1f, 0x1a, 0x9c, 0xce, 0xe0, 0x89, 0x54, 0xb2, 0xce, 0x0d, 0x1e, 0xf8, 0xe8, 0x25, - 0x18, 0x11, 0x29, 0x65, 0xad, 0xc7, 0x68, 0x62, 0x4d, 0xad, 0xe3, 0x18, 0x43, 0xf4, 0x5b, 0x6a, - 0x24, 0x2f, 0xda, 0xf5, 0x5c, 0xb6, 0xdf, 0x5a, 0x89, 0x21, 0x38, 0x85, 0x85, 0xbe, 0x0c, 0x88, - 0x11, 0xc3, 0xa6, 0xdf, 0x90, 0x9f, 0xd7, 0x0d, 0x6a, 0x07, 0x8c, 0xc8, 0x48, 0x1c, 0xa9, 0x3d, - 0xab, 0x68, 0x11, 0xee, 0xc2, 0xc0, 0x3d, 0xa8, 0xd0, 0xa7, 0x61, 0xb8, 0x4d, 0x7c, 0x5f, 0xf4, - 0x6d, 0x05, 0xa9, 0xec, 0xa4, 0x62, 0x30, 0xbc, 0x1a, 0x2e, 0xe3, 0x08, 0xae, 0xff, 0x26, 0xd7, - 0x61, 0xcc, 0x06, 0x21, 0x0c, 0x5d, 0x82, 0x71, 0x23, 0x35, 0x7f, 0xf6, 0x67, 0x34, 0xe9, 0xf4, - 0x53, 0xfb, 0x7b, 0x73, 0xe3, 0xe9, 0xc1, 0xb4, 0x8f, 0xb3, 0x78, 0x88, 0xc0, 0x08, 0xf5, 0x54, - 0x6b, 0x1c, 0x9a, 0xea, 0xd2, 0xe0, 0x97, 0xbf, 0xa4, 0x4f, 0x0e, 0x38, 0xee, 0x89, 0x63, 0xd6, - 0x68, 0x0e, 0x8a, 0xcd, 0x77, 0x2c, 0x27, 0x0a, 0xc6, 0x92, 0xb0, 0xe5, 0xf5, 0x37, 0xae, 0xad, - 0xf9, 0x38, 0x5c, 0x47, 0x5c, 0x74, 0xbc, 0x2a, 0x4f, 0x47, 0xc5, 0xcb, 0xd1, 0xb3, 0x7f, 0xaa, - 0x67, 0x8e, 0x78, 0xe3, 0x94, 0x1c, 0xd1, 0xfc, 0x3e, 0xd3, 0x3b, 0x9c, 0xd0, 0x45, 0x28, 0x88, - 0x54, 0xac, 0x9c, 0xe7, 0x5c, 0x74, 0x01, 0x6f, 0xec, 0x7a, 0xe4, 0xe1, 0xde, 0x5c, 0xd6, 0x04, - 0x62, 0x11, 0x4b, 0xf4, 0x81, 0xab, 0xf8, 0xf8, 0xa2, 0xcf, 0x1f, 0x54, 0x46, 0x14, 0x8e, 0x52, - 0x46, 0xfc, 0xa2, 0xd8, 0xe1, 0x35, 0xe2, 0xd2, 0x44, 0xaf, 0x42, 0xc9, 0xa2, 0x8c, 0x98, 0xd2, - 0xeb, 0xc3, 0x8d, 0xce, 0x46, 0xca, 0x5e, 0x8b, 0x00, 0x0f, 0xd3, 0x1f, 0x38, 0x21, 0x40, 0x26, - 0x14, 0x9a, 0xcc, 0x6d, 0xab, 0x6a, 0xf8, 0x68, 0x37, 0xba, 0x70, 0xe2, 0x64, 0xf3, 0xd7, 0x99, - 0xdb, 0xc6, 0x92, 0x39, 0xba, 0x03, 0x39, 0xee, 0xaa, 0x01, 0xda, 0x31, 0x88, 0x00, 0x25, 0x22, - 0xb7, 0xe1, 0xe2, 0x1c, 0x77, 0x85, 0xfb, 0xfb, 0x59, 0xa7, 0xbb, 0x74, 0x48, 0xa7, 0x4b, 0xdc, - 0x3f, 0xf6, 0xb4, 0x98, 0xb5, 0x1c, 0x7d, 0x76, 0x24, 0x8a, 0x24, 0x57, 0x77, 0xa5, 0x96, 0xdb, - 0x30, 0x64, 0x84, 0x36, 0x19, 0x92, 0x36, 0xf9, 0xa2, 0x1c, 0x35, 0x46, 0xc6, 0x98, 0x7f, 0xc4, - 0xc3, 0x2e, 0xb3, 0xe2, 0x67, 0xd6, 0x8a, 0xb0, 0x70, 0x48, 0x84, 0x15, 0x3b, 0x74, 0x05, 0xc6, - 0x89, 0x63, 0x6c, 0xda, 0xe4, 0x86, 0xdb, 0x6a, 0x51, 0xa7, 0x35, 0x33, 0x2c, 0x6f, 0xab, 0xa7, - 0x95, 0x2e, 0xe3, 0xcb, 0x69, 0x20, 0xce, 0xe2, 0xf6, 0xca, 0xac, 0x23, 0x03, 0x64, 0xd6, 0xc8, - 0xcf, 0x4b, 0xfd, 0xfc, 0x5c, 0xbf, 0x97, 0x07, 0x94, 0xb1, 0x98, 0xb8, 0xca, 0x7d, 0xd1, 0x7f, - 0x8d, 0x3b, 0xe9, 0x65, 0x95, 0xad, 0x8e, 0x2b, 0x6f, 0xc6, 0xbb, 0xcf, 0xc2, 0xb3, 0x32, 0x91, - 0x07, 0x63, 0x9c, 0x19, 0xcd, 0x26, 0x35, 0xa5, 0x56, 0xca, 0xe9, 0x5f, 0x79, 0x84, 0x0e, 0xf2, - 0xd5, 0xbb, 0x12, 0x9b, 0x63, 0x23, 0x45, 0x9d, 0x9a, 0x01, 0xa6, 0x56, 0x71, 0x46, 0x02, 0x7a, - 0x4f, 0x83, 0xb2, 0xa8, 0x69, 0xd2, 0x28, 0x6a, 0xac, 0xf1, 0x85, 0xc7, 0x17, 0x8b, 0x3b, 0x38, - 0x24, 0x3d, 0x76, 0x27, 0x04, 0x77, 0x49, 0xd3, 0xff, 0xac, 0xc1, 0x74, 0x97, 0x45, 0x82, 0x93, - 0x18, 0x1f, 0xdb, 0x50, 0x14, 0xc9, 0x39, 0xca, 0x49, 0x2b, 0x47, 0xb2, 0x75, 0x52, 0x16, 0x24, - 0x85, 0x84, 0x58, 0xf3, 0x71, 0x28, 0x44, 0x9f, 0x87, 0xf1, 0x4c, 0x63, 0x7a, 0xf0, 0xb4, 0x46, - 0xff, 0xa0, 0x08, 0xe5, 0x88, 0xaf, 0xbf, 0x1e, 0xb4, 0xdb, 0x06, 0x3b, 0x89, 0x32, 0xfa, 0xbb, - 0x1a, 0x4c, 0xa6, 0x1d, 0x93, 0xc6, 0x47, 0x54, 0x3b, 0xd2, 0x11, 0x85, 0xbe, 0x71, 0x5a, 0xc9, - 0x9e, 0x5c, 0xcb, 0x8a, 0xc0, 0x9d, 0x32, 0xd1, 0xaf, 0x34, 0x78, 0x3e, 0x94, 0xa2, 0x9e, 0x17, - 0x3a, 0x28, 0x94, 0xa3, 0x1e, 0x87, 0x52, 0xff, 0xaf, 0x94, 0x7a, 0x7e, 0xf1, 0x11, 0xf2, 0xf0, - 0x23, 0xb5, 0x41, 0x3f, 0xd5, 0xe0, 0xe9, 0x10, 0xa1, 0x53, 0xcf, 0xc2, 0xb1, 0xe9, 0x79, 0x46, - 0xe9, 0xf9, 0xf4, 0x62, 0x2f, 0x41, 0xb8, 0xb7, 0x7c, 0xd1, 0x10, 0xb4, 0xa3, 0x96, 0x75, 0xa6, - 0x78, 0x38, 0x65, 0xba, 0x7b, 0xde, 0xa4, 0xe6, 0x88, 0x61, 0x38, 0x91, 0xa3, 0xdf, 0x81, 0xa7, - 0x1a, 0x46, 0x8b, 0x3a, 0xb2, 0x06, 0x5d, 0x21, 0xfc, 0xa6, 0x27, 0x7e, 0xc8, 0x3b, 0xda, 0x13, - 0x35, 0xa8, 0x26, 0x0b, 0xe0, 0xa4, 0x2d, 0x15, 0x05, 0xa8, 0x84, 0x88, 0x5e, 0xda, 0xa6, 0x6d, - 0xca, 0x55, 0x8d, 0x1c, 0x87, 0xd3, 0x0d, 0xb1, 0x88, 0x43, 0x98, 0x6e, 0xc0, 0x58, 0xba, 0x1f, - 0x7e, 0x12, 0xb3, 0xcf, 0xbf, 0xe7, 0x20, 0x9a, 0xea, 0xa0, 0x0b, 0xa9, 0x46, 0x38, 0x14, 0x31, - 0x73, 0x70, 0x13, 0x8c, 0xd6, 0x54, 0x0b, 0x9e, 0x3b, 0x20, 0x4e, 0x03, 0x4e, 0xed, 0x4a, 0xf8, - 0x6f, 0x3b, 0x95, 0xba, 0xc3, 0x6f, 0xb2, 0x75, 0xce, 0xa8, 0xd3, 0x0a, 0x67, 0x57, 0xa9, 0x86, - 0xfd, 0x53, 0x30, 0x4c, 0x1c, 0xd9, 0xdd, 0xcb, 0x6a, 0xa5, 0x18, 0x4e, 0x9e, 0x96, 0xc3, 0x25, - 0x1c, 0xc1, 0x44, 0x83, 0x49, 0xcd, 0xb6, 0x27, 0x2a, 0x46, 0x59, 0xd1, 0x15, 0xc3, 0x06, 0xb3, - 0xbe, 0xb4, 0xda, 0x90, 0x55, 0x64, 0x0c, 0x8d, 0x30, 0x97, 0xa2, 0x69, 0x5b, 0x0a, 0x53, 0xac, - 0xe1, 0x18, 0x2a, 0x31, 0x5b, 0x8a, 0xe7, 0x50, 0x0a, 0x73, 0x25, 0xe6, 0xa9, 0xa0, 0xe8, 0xb2, - 0x7a, 0x7a, 0x51, 0x2d, 0x81, 0xcc, 0xff, 0xa5, 0x8e, 0xd7, 0x93, 0x68, 0x16, 0x93, 0xc1, 0xd4, - 0x09, 0x94, 0x3b, 0xab, 0xeb, 0x27, 0x61, 0xd7, 0x7b, 0x05, 0x38, 0xbd, 0x1e, 0x78, 0xe2, 0x44, - 0xc3, 0x97, 0xdc, 0x25, 0xd7, 0xb6, 0x55, 0xbd, 0xf9, 0xe4, 0x6f, 0xd7, 0x37, 0xa1, 0x44, 0xee, - 0x7a, 0x94, 0x11, 0x6b, 0x31, 0x72, 0x8c, 0xcf, 0x3c, 0x9e, 0x88, 0x0d, 0xda, 0x26, 0xc9, 0xd6, - 0x96, 0x23, 0x26, 0x38, 0xe1, 0x27, 0xce, 0xc2, 0xa7, 0x8e, 0x49, 0x04, 0xaa, 0xaa, 0xf6, 0x63, - 0x82, 0xf5, 0x08, 0x80, 0x13, 0x1c, 0xd1, 0x12, 0x35, 0xe3, 0xb7, 0x6f, 0xe9, 0x2c, 0x87, 0x68, - 0x89, 0x3a, 0xdf, 0xd0, 0x93, 0x13, 0x48, 0xd6, 0x70, 0x4a, 0x0e, 0xfa, 0xa1, 0x06, 0x13, 0x46, - 0xf6, 0xf9, 0x3a, 0x9c, 0xf5, 0xae, 0x1e, 0x4e, 0x74, 0x9f, 0xa7, 0xf8, 0xda, 0x33, 0x4a, 0x8f, - 0x89, 0x8e, 0x77, 0xec, 0x0e, 0xe1, 0xfa, 0xc7, 0x1a, 0x3c, 0xd7, 0xc7, 0x23, 0x4e, 0x60, 0x8c, - 0x61, 0x67, 0xc7, 0x18, 0x03, 0xd7, 0x21, 0x7d, 0x34, 0xef, 0x33, 0xd0, 0xf8, 0x49, 0x0e, 0xce, - 0xf5, 0xa1, 0x38, 0xf4, 0x68, 0xe3, 0x0a, 0x8c, 0x47, 0xbf, 0xd3, 0x61, 0x98, 0x54, 0xbd, 0x69, - 0x20, 0xce, 0xe2, 0x46, 0xa2, 0xe4, 0xcd, 0x92, 0xef, 0x16, 0x15, 0xde, 0x2e, 0x11, 0x86, 0xf0, - 0x70, 0xd3, 0x6d, 0x7b, 0x36, 0xe1, 0x24, 0x6c, 0x57, 0x47, 0x12, 0x0f, 0x5f, 0x8a, 0x00, 0x38, - 0xc1, 0x11, 0xd9, 0x84, 0x30, 0xe6, 0x32, 0xe9, 0x61, 0xa9, 0xc9, 0xec, 0xb2, 0x58, 0xc4, 0x21, - 0x4c, 0xff, 0xa7, 0x06, 0x67, 0xfa, 0x1c, 0xca, 0x89, 0x95, 0xa3, 0x3b, 0xd9, 0x72, 0xf4, 0x8d, - 0x63, 0x72, 0x83, 0x83, 0x0a, 0xd3, 0xda, 0xc6, 0xfd, 0x07, 0xb3, 0xa7, 0x3e, 0x7c, 0x30, 0x7b, - 0xea, 0xa3, 0x07, 0xb3, 0xa7, 0xde, 0xdb, 0x9f, 0xd5, 0xee, 0xef, 0xcf, 0x6a, 0x1f, 0xee, 0xcf, - 0x6a, 0x1f, 0xed, 0xcf, 0x6a, 0x7f, 0xdc, 0x9f, 0xd5, 0x7e, 0xfc, 0xa7, 0xd9, 0x53, 0x5f, 0xa9, - 0x0c, 0xf6, 0xaf, 0xbe, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x2b, 0xe3, 0x29, 0x87, 0x1b, 0x2c, - 0x00, 0x00, + // 2632 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x1a, 0x4b, 0x73, 0x1c, 0x47, + 0xd9, 0xb3, 0x0f, 0x49, 0xfb, 0xe9, 0xb5, 0x6a, 0x25, 0xb1, 0x48, 0x62, 0xc9, 0x99, 0x40, 0xca, + 0x50, 0x61, 0x37, 0x12, 0x71, 0x6c, 0x70, 0x1c, 0xd0, 0xca, 0xb2, 0x6a, 0x89, 0x24, 0x6f, 0x5a, + 0x72, 0xb9, 0x8a, 0xe0, 0x90, 0xd1, 0x4c, 0xef, 0xaa, 0xd1, 0xec, 0xcc, 0xa4, 0xa7, 0x47, 0xb1, + 0x38, 0x50, 0xa1, 0x80, 0x43, 0x78, 0x99, 0xe2, 0x42, 0x71, 0x83, 0x13, 0x17, 0x7e, 0x41, 0x6e, + 0xdc, 0x7c, 0x0c, 0x45, 0x51, 0xe4, 0xa4, 0xc2, 0xa2, 0x80, 0xca, 0x01, 0x0e, 0xdc, 0x30, 0x17, + 0xaa, 0x7b, 0x7a, 0x5e, 0xfb, 0xb0, 0xbc, 0x92, 0x2c, 0xaa, 0x20, 0x27, 0xed, 0xf4, 0xf7, 0xec, + 0xfe, 0xbe, 0xaf, 0xbf, 0x47, 0x0b, 0x5e, 0x33, 0x1c, 0xce, 0x88, 0x51, 0xa1, 0x6e, 0x35, 0xfc, + 0x55, 0xf5, 0x76, 0x5a, 0x55, 0xc3, 0xa3, 0x7e, 0xd5, 0x74, 0x1d, 0xce, 0x5c, 0xdb, 0xb3, 0x0d, + 0x87, 0x54, 0x77, 0xe7, 0xb7, 0x08, 0x37, 0x16, 0xaa, 0x2d, 0xe2, 0x10, 0x66, 0x70, 0x62, 0x55, + 0x3c, 0xe6, 0x72, 0x17, 0x55, 0x42, 0xaa, 0x6f, 0x50, 0x57, 0xfd, 0xaa, 0x78, 0x3b, 0xad, 0x8a, + 0xa0, 0xaf, 0xa4, 0xe9, 0x2b, 0x8a, 0xfe, 0xe9, 0xcb, 0xfd, 0xe5, 0xf9, 0xdc, 0xe0, 0x7e, 0x75, + 0x77, 0xde, 0xb0, 0xbd, 0x6d, 0x63, 0xbe, 0x53, 0xd2, 0xd3, 0x9f, 0x6f, 0x51, 0xbe, 0x1d, 0x6c, + 0x55, 0x4c, 0xb7, 0x5d, 0x6d, 0xb9, 0x2d, 0xb7, 0x2a, 0x97, 0xb7, 0x82, 0xa6, 0xfc, 0x92, 0x1f, + 0xf2, 0x97, 0x42, 0x7f, 0x79, 0xe7, 0xb2, 0x2f, 0xa5, 0x78, 0xb4, 0x6d, 0x98, 0xdb, 0xd4, 0x21, + 0x6c, 0x2f, 0x91, 0xd5, 0x26, 0xdc, 0xa8, 0xee, 0x76, 0x0b, 0xa9, 0xf6, 0xa3, 0x62, 0x81, 0xc3, + 0x69, 0x9b, 0x74, 0x11, 0xbc, 0x72, 0x18, 0x81, 0x6f, 0x6e, 0x93, 0xb6, 0xd1, 0x45, 0xf7, 0x85, + 0x7e, 0x74, 0x01, 0xa7, 0x76, 0x95, 0x3a, 0xdc, 0xe7, 0xac, 0x93, 0x48, 0xff, 0x9b, 0x06, 0x63, + 0x8b, 0x96, 0xc5, 0x88, 0xef, 0xaf, 0x30, 0x37, 0xf0, 0xd0, 0xdb, 0x30, 0x22, 0x76, 0x62, 0x19, + 0xdc, 0x98, 0xd1, 0xce, 0x6b, 0x17, 0x46, 0x17, 0x5e, 0xaa, 0x84, 0x8c, 0x2b, 0x69, 0xc6, 0x89, + 0x4d, 0x04, 0x76, 0x65, 0x77, 0xbe, 0x72, 0x63, 0xeb, 0x9b, 0xc4, 0xe4, 0x6b, 0x84, 0x1b, 0x35, + 0x74, 0x6f, 0x7f, 0xee, 0xcc, 0xc1, 0xfe, 0x1c, 0x24, 0x6b, 0x38, 0xe6, 0x8a, 0x02, 0x18, 0x6b, + 0x09, 0x51, 0x6b, 0xa4, 0xbd, 0x45, 0x98, 0x3f, 0x93, 0x3b, 0x9f, 0xbf, 0x30, 0xba, 0x70, 0x65, + 0x40, 0xb3, 0x57, 0x56, 0x12, 0x1e, 0xb5, 0x27, 0x94, 0xc0, 0xb1, 0xd4, 0xa2, 0x8f, 0x33, 0x62, + 0xf4, 0xdf, 0x6b, 0x50, 0x4e, 0xef, 0x74, 0x95, 0xfa, 0x1c, 0x7d, 0xbd, 0x6b, 0xb7, 0x95, 0x47, + 0xdb, 0xad, 0xa0, 0x96, 0x7b, 0x2d, 0x2b, 0xd1, 0x23, 0xd1, 0x4a, 0x6a, 0xa7, 0x06, 0x14, 0x29, + 0x27, 0xed, 0x68, 0x8b, 0xaf, 0x0e, 0xba, 0xc5, 0xb4, 0xba, 0xb5, 0x71, 0x25, 0xa8, 0x58, 0x17, + 0x2c, 0x71, 0xc8, 0x59, 0x7f, 0x3f, 0x0f, 0x53, 0x69, 0xb4, 0x86, 0xc1, 0xcd, 0xed, 0x53, 0x30, + 0xe2, 0xf7, 0x34, 0x98, 0x32, 0x2c, 0x8b, 0x58, 0x2b, 0x27, 0x6c, 0xca, 0x4f, 0x29, 0xb1, 0x62, + 0x57, 0x59, 0xee, 0xb8, 0x5b, 0x20, 0xfa, 0x81, 0x06, 0xd3, 0x8c, 0xb4, 0xdd, 0xdd, 0x0e, 0x45, + 0xf2, 0xc7, 0x57, 0xe4, 0x19, 0xa5, 0xc8, 0x34, 0xee, 0xe6, 0x8f, 0x7b, 0x09, 0xd5, 0x3f, 0xd6, + 0x60, 0x62, 0xd1, 0xf3, 0x6c, 0x4a, 0xac, 0x4d, 0xf7, 0x7f, 0x3c, 0x9a, 0xfe, 0xa8, 0x01, 0xca, + 0xee, 0xf5, 0x14, 0xe2, 0xc9, 0xcc, 0xc6, 0xd3, 0x6b, 0x03, 0xc7, 0x53, 0x46, 0xe1, 0x3e, 0x11, + 0xf5, 0xc3, 0x3c, 0x4c, 0x67, 0x11, 0x3f, 0x89, 0xa9, 0xff, 0x5e, 0x4c, 0xbd, 0x03, 0xd3, 0x35, + 0xc3, 0xa7, 0xe6, 0x62, 0xc0, 0xb7, 0x89, 0xc3, 0xa9, 0x69, 0x70, 0xea, 0x3a, 0xe8, 0x45, 0x18, + 0x09, 0x7c, 0xc2, 0x1c, 0xa3, 0x4d, 0xa4, 0x31, 0x4a, 0x89, 0xdf, 0xdc, 0x54, 0xeb, 0x38, 0xc6, + 0x10, 0xd8, 0x9e, 0xe1, 0xfb, 0xef, 0xba, 0xcc, 0x9a, 0xc9, 0x65, 0xb1, 0x1b, 0x6a, 0x1d, 0xc7, + 0x18, 0xfa, 0x3c, 0x94, 0x6b, 0x81, 0x63, 0xd9, 0xe4, 0x3a, 0xb5, 0xc9, 0x06, 0x61, 0xbb, 0x84, + 0xa1, 0x73, 0x90, 0x0f, 0x98, 0xad, 0x44, 0x8d, 0x2a, 0xe2, 0xfc, 0x4d, 0xbc, 0x8a, 0xc5, 0xba, + 0x7e, 0x37, 0x07, 0xe7, 0x42, 0x9a, 0x10, 0x5f, 0x68, 0xbb, 0xe4, 0x3a, 0x4d, 0xda, 0x0a, 0x58, + 0xa8, 0xf0, 0x45, 0x18, 0xdd, 0x22, 0x06, 0x23, 0x6c, 0xd3, 0xdd, 0x21, 0x8e, 0x62, 0x34, 0xad, + 0x18, 0x8d, 0xd6, 0x12, 0x10, 0x4e, 0xe3, 0xa1, 0x17, 0x60, 0xc8, 0xf0, 0xe8, 0xeb, 0x64, 0x4f, + 0xe9, 0x3d, 0xa1, 0x28, 0x86, 0x16, 0x1b, 0xf5, 0xd7, 0xc9, 0x1e, 0x56, 0x50, 0xf4, 0x13, 0x0d, + 0xa6, 0xb7, 0xba, 0xcf, 0x69, 0x26, 0x2f, 0x1d, 0x75, 0x69, 0x50, 0x9b, 0xf5, 0x38, 0xf2, 0xda, + 0x59, 0x61, 0xb7, 0x1e, 0x00, 0xdc, 0x4b, 0xb0, 0xfe, 0xcb, 0x02, 0x4c, 0x2f, 0xd9, 0x81, 0xcf, + 0x09, 0xcb, 0x38, 0xd7, 0xe3, 0x8f, 0xa2, 0xef, 0x68, 0x50, 0x26, 0xcd, 0x26, 0x31, 0x39, 0xdd, + 0x25, 0x27, 0x18, 0x44, 0x33, 0x4a, 0x6a, 0x79, 0xb9, 0x83, 0x39, 0xee, 0x12, 0x87, 0xbe, 0x0d, + 0x53, 0xf1, 0x5a, 0xbd, 0x51, 0xb3, 0x5d, 0x73, 0x27, 0x8a, 0x9f, 0x8b, 0x83, 0xea, 0x50, 0x6f, + 0xac, 0x13, 0x9e, 0x84, 0xf0, 0x72, 0x27, 0x5f, 0xdc, 0x2d, 0x0a, 0x5d, 0x86, 0x31, 0xee, 0x72, + 0xc3, 0x8e, 0xb6, 0x5f, 0x38, 0xaf, 0x5d, 0xc8, 0x27, 0xf7, 0xfa, 0x66, 0x0a, 0x86, 0x33, 0x98, + 0x68, 0x01, 0x40, 0x7e, 0x37, 0x8c, 0x16, 0xf1, 0x67, 0x8a, 0x92, 0x2e, 0x3e, 0xef, 0xcd, 0x18, + 0x82, 0x53, 0x58, 0xc2, 0xb7, 0xcd, 0x80, 0x31, 0xe2, 0x70, 0xf1, 0x3d, 0x33, 0x24, 0x89, 0x62, + 0xdf, 0x5e, 0x4a, 0x40, 0x38, 0x8d, 0xa7, 0xff, 0x55, 0x83, 0xd1, 0xe5, 0xd6, 0xff, 0x41, 0xe5, + 0xf9, 0x3b, 0x0d, 0x26, 0x53, 0x1b, 0x3d, 0x85, 0x44, 0xf9, 0x76, 0x36, 0x51, 0x0e, 0xbc, 0xc3, + 0x94, 0xb6, 0x7d, 0xb2, 0xe4, 0x8f, 0xf2, 0x50, 0x4e, 0x61, 0x85, 0x29, 0xd2, 0x02, 0x70, 0xe3, + 0x73, 0x3f, 0x51, 0x1b, 0xa6, 0xf8, 0x7e, 0x92, 0x26, 0x7b, 0xa4, 0x49, 0x1b, 0xce, 0x2e, 0xdf, + 0xe1, 0x22, 0xdd, 0xd9, 0xcb, 0x0e, 0xa7, 0x7c, 0x0f, 0x93, 0x26, 0x61, 0xc4, 0x31, 0x09, 0x3a, + 0x0f, 0x85, 0x54, 0x9a, 0x1c, 0x53, 0xac, 0x0b, 0xeb, 0x22, 0x45, 0x4a, 0x08, 0xaa, 0x42, 0x49, + 0xfc, 0xf5, 0x3d, 0xc3, 0x24, 0x2a, 0xcf, 0x4c, 0x29, 0xb4, 0xd2, 0x7a, 0x04, 0xc0, 0x09, 0x8e, + 0xfe, 0x6f, 0x0d, 0xca, 0x52, 0xfc, 0xa2, 0xef, 0xbb, 0x26, 0x0d, 0x33, 0xdc, 0xa9, 0xd4, 0x47, + 0x65, 0x43, 0x49, 0x54, 0xfb, 0x3f, 0x72, 0x29, 0x28, 0xa9, 0xe3, 0x43, 0x4a, 0x2e, 0xf7, 0xc5, + 0x0e, 0xfe, 0xb8, 0x4b, 0xa2, 0xfe, 0x41, 0x01, 0x46, 0x53, 0x87, 0x8f, 0x6e, 0x41, 0xde, 0x73, + 0x2d, 0xb5, 0xe7, 0x81, 0x7b, 0xbc, 0x86, 0x6b, 0x25, 0x6a, 0x0c, 0x8b, 0xaa, 0x42, 0xac, 0x08, + 0x8e, 0xe8, 0xbb, 0x1a, 0x4c, 0x90, 0x8c, 0x55, 0xa5, 0x75, 0x46, 0x17, 0x56, 0x06, 0x8e, 0xe7, + 0xde, 0xbe, 0x51, 0x43, 0x07, 0xfb, 0x73, 0x13, 0x1d, 0xc0, 0x0e, 0x91, 0xe8, 0x05, 0xc8, 0x53, + 0x2f, 0x74, 0xeb, 0xb1, 0xda, 0x13, 0x42, 0xc1, 0x7a, 0xc3, 0x7f, 0xb0, 0x3f, 0x57, 0xaa, 0x37, + 0x54, 0xe3, 0x89, 0x05, 0x02, 0x7a, 0x0b, 0x8a, 0x9e, 0xcb, 0xb8, 0x48, 0x36, 0xc2, 0x22, 0x5f, + 0x1c, 0x54, 0x47, 0xe1, 0x69, 0x56, 0xc3, 0x65, 0x3c, 0xb9, 0x71, 0xc4, 0x97, 0x8f, 0x43, 0xb6, + 0xe8, 0x4d, 0x28, 0x38, 0xae, 0x45, 0x64, 0x4e, 0x1a, 0x5d, 0xb8, 0x3a, 0x30, 0x7b, 0xd7, 0x22, + 0xc9, 0xc6, 0x47, 0x64, 0x08, 0x88, 0x25, 0xc9, 0x14, 0xb5, 0x60, 0xd8, 0x27, 0x6c, 0x97, 0x9a, + 0x61, 0xfa, 0x1a, 0x5d, 0xf8, 0xca, 0xa0, 0xfc, 0x37, 0x42, 0xf2, 0x44, 0xc4, 0xe8, 0xc1, 0xfe, + 0xdc, 0x70, 0xb4, 0x1a, 0x71, 0xd7, 0x7f, 0xad, 0xc1, 0x44, 0xd6, 0xf7, 0xb2, 0xe1, 0xa7, 0x1d, + 0x1e, 0x7e, 0x71, 0x44, 0xe7, 0xfa, 0x46, 0x74, 0x0d, 0xf2, 0x01, 0xb5, 0x64, 0xf5, 0x57, 0xaa, + 0xbd, 0x14, 0x97, 0xab, 0xf5, 0x6b, 0x0f, 0xf6, 0xe7, 0x9e, 0xeb, 0x37, 0x26, 0xe2, 0x7b, 0x1e, + 0xf1, 0x2b, 0x37, 0xeb, 0xd7, 0xb0, 0x20, 0xd6, 0x7f, 0xab, 0xc1, 0xb0, 0x2a, 0x28, 0xd0, 0x2d, + 0x28, 0x98, 0xd4, 0x62, 0xca, 0xc7, 0x8f, 0x58, 0xc2, 0xc4, 0x8a, 0x2e, 0xd5, 0xaf, 0x61, 0x2c, + 0x19, 0xa2, 0xdb, 0x30, 0x44, 0xee, 0x98, 0xc4, 0xe3, 0x2a, 0x8e, 0x8f, 0xc8, 0x3a, 0x2e, 0x8b, + 0x97, 0x25, 0x33, 0xac, 0x98, 0xea, 0x4d, 0x28, 0x4a, 0x04, 0xf4, 0x3c, 0xe4, 0xa8, 0x27, 0xd5, + 0x1f, 0xab, 0x4d, 0x1f, 0xec, 0xcf, 0xe5, 0xea, 0x8d, 0xac, 0x0b, 0xe7, 0xa8, 0x27, 0xaa, 0x26, + 0x8f, 0x91, 0x26, 0xbd, 0xb3, 0x4a, 0x9c, 0x16, 0xdf, 0x96, 0xe7, 0x5b, 0x4c, 0x32, 0x7c, 0x23, + 0x05, 0xc3, 0x19, 0x4c, 0xfd, 0x17, 0x1a, 0xa0, 0xb5, 0xc0, 0x16, 0xd5, 0xaf, 0xcf, 0xa5, 0x79, + 0xeb, 0x4e, 0xd3, 0x45, 0xcf, 0x43, 0x51, 0x16, 0x02, 0xca, 0xaa, 0xb1, 0x5f, 0x87, 0x0e, 0x10, + 0xc2, 0xd0, 0x5b, 0x50, 0xf0, 0x5c, 0xeb, 0xc8, 0x33, 0xa2, 0xcc, 0xfd, 0x11, 0x1f, 0x71, 0xc3, + 0xb5, 0x7c, 0x2c, 0xf9, 0xea, 0xef, 0x6b, 0x50, 0x8a, 0x63, 0x4b, 0xf8, 0x8e, 0x08, 0x27, 0xa9, + 0x51, 0x31, 0x8d, 0xcf, 0x38, 0x96, 0x90, 0x47, 0xf0, 0xae, 0xcb, 0x30, 0x22, 0x87, 0x87, 0xa6, + 0x6b, 0x2b, 0x17, 0x7b, 0x36, 0x6e, 0xa7, 0xd4, 0xfa, 0x83, 0xd4, 0x6f, 0x1c, 0x63, 0xeb, 0x7f, + 0xcf, 0xc3, 0xf8, 0x3a, 0xe1, 0xef, 0xba, 0x6c, 0xa7, 0xe1, 0xda, 0xd4, 0xdc, 0x3b, 0x85, 0xac, + 0xd1, 0x84, 0x22, 0x0b, 0x6c, 0x12, 0x1d, 0xf0, 0xe2, 0xc0, 0x17, 0x47, 0x5a, 0x5f, 0x1c, 0xd8, + 0x24, 0xb1, 0xa3, 0xf8, 0xf2, 0x71, 0xc8, 0x1e, 0x5d, 0x85, 0x49, 0x23, 0x33, 0x36, 0x08, 0xef, + 0xcc, 0x92, 0xf4, 0xb7, 0xc9, 0xec, 0x44, 0xc1, 0xc7, 0x9d, 0xb8, 0xe8, 0x82, 0x38, 0x54, 0xea, + 0x32, 0x71, 0xcb, 0x8b, 0x72, 0x5d, 0xab, 0x8d, 0x85, 0x07, 0x1a, 0xae, 0xe1, 0x18, 0x8a, 0x5e, + 0x86, 0x31, 0x4e, 0x09, 0x8b, 0x20, 0xf2, 0x42, 0x2c, 0xd6, 0xca, 0xb2, 0xb0, 0x4f, 0xad, 0xe3, + 0x0c, 0x16, 0xf2, 0xa1, 0xe4, 0xbb, 0x01, 0x93, 0x37, 0x94, 0xba, 0xe3, 0xae, 0x1f, 0xef, 0x28, + 0x62, 0xaf, 0x1b, 0x17, 0x37, 0xd5, 0x46, 0xc4, 0x1c, 0x27, 0x72, 0xf4, 0x3f, 0x68, 0x30, 0x95, + 0x21, 0x3a, 0x85, 0xda, 0x77, 0x2b, 0x5b, 0xfb, 0x5e, 0x3d, 0xd6, 0x26, 0xfb, 0x54, 0xbf, 0xff, + 0xd4, 0xe0, 0x6c, 0x06, 0x4f, 0xa4, 0x92, 0x0d, 0x6e, 0xf0, 0xc0, 0x47, 0x2f, 0xc2, 0x88, 0x48, + 0x29, 0xeb, 0x3d, 0x46, 0x13, 0xeb, 0x6a, 0x1d, 0xc7, 0x18, 0xa2, 0xdf, 0x52, 0x23, 0x79, 0xd1, + 0xae, 0xe7, 0xb2, 0xfd, 0xd6, 0x4a, 0x0c, 0xc1, 0x29, 0x2c, 0xf4, 0x55, 0x40, 0x8c, 0x18, 0x36, + 0xfd, 0x96, 0xfc, 0xbc, 0x6e, 0x50, 0x3b, 0x60, 0x44, 0x46, 0xe2, 0x48, 0xed, 0x69, 0x45, 0x8b, + 0x70, 0x17, 0x06, 0xee, 0x41, 0x85, 0x3e, 0x0b, 0xc3, 0x6d, 0xe2, 0xfb, 0xa2, 0x6f, 0x2b, 0x48, + 0x65, 0x27, 0x15, 0x83, 0xe1, 0xb5, 0x70, 0x19, 0x47, 0x70, 0x39, 0x6a, 0xce, 0x6c, 0xba, 0x41, + 0x08, 0x43, 0x97, 0x60, 0xdc, 0x48, 0xcd, 0x9f, 0xfd, 0x19, 0x4d, 0x3a, 0xfd, 0xd4, 0xc1, 0xfe, + 0xdc, 0x78, 0x7a, 0x30, 0xed, 0xe3, 0x2c, 0x1e, 0x22, 0x30, 0x42, 0x3d, 0xd5, 0x1a, 0x87, 0xa6, + 0xba, 0x34, 0xf8, 0xe5, 0x2f, 0xe9, 0x93, 0x03, 0x8e, 0x7b, 0xe2, 0x98, 0x35, 0x9a, 0x83, 0x62, + 0xf3, 0x1d, 0xcb, 0x89, 0x82, 0xb1, 0x24, 0x6c, 0x79, 0xfd, 0x8d, 0x6b, 0xeb, 0x3e, 0x0e, 0xd7, + 0x11, 0x17, 0x1d, 0xaf, 0xca, 0xd3, 0x51, 0xf1, 0x72, 0xfc, 0xec, 0x9f, 0xea, 0x99, 0x23, 0xde, + 0x38, 0x25, 0x47, 0xdc, 0x16, 0xb6, 0xb1, 0x45, 0xec, 0xba, 0x45, 0x44, 0x99, 0x45, 0x65, 0xb3, + 0x9d, 0xbf, 0x30, 0x1e, 0xde, 0x16, 0xab, 0x59, 0x10, 0xee, 0xc4, 0x15, 0xbd, 0xf3, 0x53, 0xbd, + 0xa3, 0x11, 0x5d, 0x84, 0x82, 0xc8, 0xe4, 0xca, 0xf7, 0x9e, 0x8b, 0xee, 0xef, 0xcd, 0x3d, 0x8f, + 0x3c, 0xd8, 0x9f, 0xcb, 0x5a, 0x50, 0x2c, 0x62, 0x89, 0x3e, 0x70, 0x13, 0x10, 0xe7, 0x89, 0xfc, + 0x61, 0x55, 0x48, 0xe1, 0x38, 0x55, 0xc8, 0xaf, 0x8a, 0x1d, 0x4e, 0x27, 0xee, 0x5c, 0xf4, 0x2a, + 0x94, 0x2c, 0xca, 0x88, 0x29, 0x83, 0x26, 0xdc, 0xe8, 0x6c, 0xa4, 0xec, 0xb5, 0x08, 0xf0, 0x20, + 0xfd, 0x81, 0x13, 0x02, 0x64, 0x42, 0xa1, 0xc9, 0xdc, 0xb6, 0x2a, 0xa6, 0x8f, 0x97, 0x10, 0x44, + 0x0c, 0x24, 0x9b, 0xbf, 0xce, 0xdc, 0x36, 0x96, 0xcc, 0xd1, 0x6d, 0xc8, 0x71, 0x57, 0xcd, 0xdf, + 0x4e, 0x40, 0x04, 0x28, 0x11, 0xb9, 0x4d, 0x17, 0xe7, 0xb8, 0x2b, 0xa2, 0xc7, 0xcf, 0xfa, 0xec, + 0xa5, 0x23, 0xfa, 0x6c, 0x12, 0x3d, 0xb1, 0xa3, 0xc6, 0xac, 0xe5, 0xe4, 0xb4, 0x23, 0xcf, 0x24, + 0xa9, 0xbe, 0x2b, 0x33, 0xdd, 0x82, 0x21, 0x23, 0xb4, 0xc9, 0x90, 0xb4, 0xc9, 0x97, 0xe5, 0xa4, + 0x32, 0x32, 0xc6, 0xfc, 0x43, 0xde, 0x85, 0x99, 0x15, 0xbf, 0xd2, 0x56, 0x84, 0x85, 0x43, 0x22, + 0xac, 0xd8, 0xa1, 0x2b, 0x30, 0x4e, 0x1c, 0x63, 0xcb, 0x26, 0xab, 0x6e, 0xab, 0x45, 0x9d, 0xd6, + 0xcc, 0xb0, 0xbc, 0xec, 0x9e, 0x54, 0xba, 0x8c, 0x2f, 0xa7, 0x81, 0x38, 0x8b, 0xdb, 0x2b, 0x31, + 0x8f, 0x0c, 0x90, 0x98, 0x23, 0x3f, 0x2f, 0xf5, 0xf3, 0x73, 0xfd, 0x6e, 0x1e, 0x50, 0xc6, 0x62, + 0x22, 0x13, 0xf8, 0xa2, 0x7d, 0x1b, 0x77, 0xd2, 0xcb, 0x2a, 0xd9, 0x9d, 0x54, 0xda, 0x8d, 0x77, + 0x9f, 0x85, 0x67, 0x65, 0x22, 0x0f, 0xc6, 0x38, 0x33, 0x9a, 0x4d, 0x6a, 0x4a, 0xad, 0x94, 0xd3, + 0xbf, 0xf2, 0x10, 0x1d, 0xe4, 0xa3, 0x79, 0x25, 0x36, 0xc7, 0x66, 0x8a, 0x3a, 0x35, 0x42, 0x4c, + 0xad, 0xe2, 0x8c, 0x04, 0xf4, 0x9e, 0x06, 0x65, 0x51, 0x12, 0xa5, 0x51, 0xd4, 0x54, 0xe4, 0x4b, + 0x8f, 0x2e, 0x16, 0x77, 0x70, 0x48, 0x5a, 0xf4, 0x4e, 0x08, 0xee, 0x92, 0xa6, 0xff, 0x45, 0x83, + 0xe9, 0x2e, 0x8b, 0x04, 0xa7, 0x31, 0x7d, 0xb6, 0xa1, 0x28, 0x72, 0x7b, 0x94, 0xd2, 0x56, 0x8e, + 0x65, 0xeb, 0xa4, 0xaa, 0x48, 0xea, 0x10, 0xb1, 0xe6, 0xe3, 0x50, 0x88, 0x3e, 0x0f, 0xe3, 0x99, + 0xbe, 0xf6, 0xf0, 0x61, 0x8f, 0xfe, 0x41, 0x11, 0xca, 0x11, 0x5f, 0x7f, 0x23, 0x68, 0xb7, 0x0d, + 0x76, 0x1a, 0x55, 0xf8, 0xf7, 0x35, 0x98, 0x4c, 0x3b, 0x26, 0x8d, 0x8f, 0xa8, 0x76, 0xac, 0x23, + 0x0a, 0x7d, 0xe3, 0xac, 0x92, 0x3d, 0xb9, 0x9e, 0x15, 0x81, 0x3b, 0x65, 0xa2, 0xdf, 0x68, 0xf0, + 0x6c, 0x28, 0x45, 0xbd, 0x4e, 0x74, 0x50, 0x28, 0x47, 0x3d, 0x09, 0xa5, 0x3e, 0xad, 0x94, 0x7a, + 0x76, 0xf1, 0x21, 0xf2, 0xf0, 0x43, 0xb5, 0x41, 0x3f, 0xd7, 0xe0, 0xc9, 0x10, 0xa1, 0x53, 0xcf, + 0xc2, 0x89, 0xe9, 0x79, 0x4e, 0xe9, 0xf9, 0xe4, 0x62, 0x2f, 0x41, 0xb8, 0xb7, 0x7c, 0xd1, 0x4f, + 0xb4, 0xa3, 0x8e, 0x57, 0x96, 0x2e, 0x47, 0x50, 0xa6, 0xbb, 0x65, 0x4e, 0x6a, 0x8e, 0x18, 0x86, + 0x13, 0x39, 0xfa, 0x6d, 0x78, 0xa2, 0x61, 0xb4, 0xa8, 0x23, 0x4b, 0xd8, 0x15, 0xc2, 0x6f, 0x78, + 0xe2, 0x87, 0xbc, 0xa3, 0x3d, 0x51, 0xc2, 0x6a, 0xb2, 0x7e, 0x4e, 0xba, 0x5a, 0x51, 0xbf, 0x4a, + 0x88, 0x68, 0xc5, 0x6d, 0xda, 0xa6, 0x5c, 0x95, 0xd8, 0x71, 0x38, 0xad, 0x8a, 0x45, 0x1c, 0xc2, + 0x74, 0x03, 0xc6, 0xd2, 0xed, 0xf4, 0xe3, 0x18, 0x9d, 0xfe, 0x23, 0x07, 0xd1, 0x50, 0x08, 0xbd, + 0x9c, 0xea, 0xa3, 0x43, 0x11, 0x33, 0x87, 0xf7, 0xd0, 0x68, 0x5d, 0x75, 0xf0, 0xb9, 0x43, 0xe2, + 0x34, 0xe0, 0xd4, 0xae, 0x84, 0xff, 0xf5, 0x53, 0xa9, 0x3b, 0xfc, 0x06, 0xdb, 0xe0, 0x8c, 0x3a, + 0xad, 0x70, 0xf4, 0x95, 0xea, 0xf7, 0x3f, 0x03, 0xc3, 0xc4, 0x91, 0xc3, 0x01, 0x59, 0xad, 0x14, + 0xc3, 0xc1, 0xd5, 0x72, 0xb8, 0x84, 0x23, 0x98, 0xe8, 0x4f, 0xa9, 0xd9, 0xf6, 0x44, 0xc5, 0x28, + 0x2b, 0xba, 0x62, 0xd8, 0x9f, 0xd6, 0x97, 0xd6, 0x1a, 0xb2, 0x8a, 0x8c, 0xa1, 0x11, 0xe6, 0x52, + 0x34, 0xac, 0x4b, 0x61, 0x8a, 0x35, 0x1c, 0x43, 0x25, 0x66, 0x4b, 0xf1, 0x1c, 0x4a, 0x61, 0xae, + 0xc4, 0x3c, 0x15, 0x14, 0x5d, 0x56, 0x2f, 0x37, 0xaa, 0xa3, 0x90, 0xf9, 0xbf, 0xd4, 0xf1, 0xf8, + 0x12, 0x8d, 0x72, 0x32, 0x98, 0x3a, 0x81, 0x72, 0x67, 0x71, 0xfe, 0x38, 0xec, 0x7a, 0xb7, 0x00, + 0x67, 0x37, 0x02, 0x4f, 0x9c, 0x68, 0xf8, 0x10, 0xbc, 0xe4, 0xda, 0xb6, 0xaa, 0x37, 0x1f, 0xff, + 0xed, 0xfa, 0x26, 0x94, 0xc8, 0x1d, 0x8f, 0x32, 0x62, 0x2d, 0x46, 0x8e, 0xf1, 0xb9, 0x47, 0x13, + 0xb1, 0x49, 0xdb, 0x24, 0xd9, 0xda, 0x72, 0xc4, 0x04, 0x27, 0xfc, 0xc4, 0x59, 0xf8, 0xd4, 0x31, + 0x89, 0x40, 0x55, 0xd5, 0x7e, 0x4c, 0xb0, 0x11, 0x01, 0x70, 0x82, 0x23, 0x3a, 0xaa, 0x66, 0xfc, + 0x74, 0x2e, 0x9d, 0xe5, 0x08, 0x1d, 0x55, 0xe7, 0x13, 0x7c, 0x72, 0x02, 0xc9, 0x1a, 0x4e, 0xc9, + 0x41, 0x3f, 0xd6, 0x60, 0xc2, 0xc8, 0xbe, 0x7e, 0x87, 0xa3, 0xe2, 0xb5, 0xa3, 0x89, 0xee, 0xf3, + 0x92, 0x5f, 0x7b, 0x4a, 0xe9, 0x31, 0xd1, 0xf1, 0x0c, 0xde, 0x21, 0x5c, 0xff, 0x58, 0x83, 0x67, + 0xfa, 0x78, 0xc4, 0x29, 0x4c, 0x41, 0xec, 0xec, 0x14, 0x64, 0xe0, 0x3a, 0xa4, 0x8f, 0xe6, 0x7d, + 0xe6, 0x21, 0x3f, 0xcb, 0xc1, 0x73, 0x7d, 0x28, 0x8e, 0x3c, 0x19, 0xb9, 0x02, 0xe3, 0xd1, 0xef, + 0x74, 0x18, 0x26, 0x55, 0x6f, 0x1a, 0x88, 0xb3, 0xb8, 0x91, 0x28, 0x79, 0xb3, 0xe4, 0xbb, 0x45, + 0x85, 0xb7, 0x4b, 0x84, 0x21, 0x3c, 0xdc, 0x74, 0xdb, 0x9e, 0x4d, 0x38, 0x09, 0xdb, 0xd5, 0x91, + 0xc4, 0xc3, 0x97, 0x22, 0x00, 0x4e, 0x70, 0x44, 0x36, 0x21, 0x8c, 0xb9, 0x4c, 0x7a, 0x58, 0x6a, + 0xb0, 0xbb, 0x2c, 0x16, 0x71, 0x08, 0xd3, 0xff, 0xa5, 0xc1, 0xb9, 0x3e, 0x87, 0x72, 0x6a, 0xe5, + 0xe8, 0x6e, 0xb6, 0x1c, 0x7d, 0xe3, 0x84, 0xdc, 0xe0, 0xb0, 0xc2, 0xb4, 0xb6, 0x79, 0xef, 0xfe, + 0xec, 0x99, 0x0f, 0xef, 0xcf, 0x9e, 0xf9, 0xe8, 0xfe, 0xec, 0x99, 0xf7, 0x0e, 0x66, 0xb5, 0x7b, + 0x07, 0xb3, 0xda, 0x87, 0x07, 0xb3, 0xda, 0x47, 0x07, 0xb3, 0xda, 0x9f, 0x0e, 0x66, 0xb5, 0x9f, + 0xfe, 0x79, 0xf6, 0xcc, 0xd7, 0x2a, 0x83, 0xfd, 0xa7, 0xf0, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, + 0x0e, 0x8b, 0x84, 0xb6, 0x5a, 0x2c, 0x00, 0x00, } func (m *AddressGroup) Marshal() (dAtA []byte, err error) { @@ -2554,6 +2555,13 @@ func (m *NetworkPolicyPeer) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.LabelIdentities) > 0 { + for iNdEx := len(m.LabelIdentities) - 1; iNdEx >= 0; iNdEx-- { + i = encodeVarintGenerated(dAtA, i, uint64(m.LabelIdentities[iNdEx])) + i-- + dAtA[i] = 0x28 + } + } if len(m.ToServices) > 0 { for iNdEx := len(m.ToServices) - 1; iNdEx >= 0; iNdEx-- { { @@ -3824,6 +3832,11 @@ func (m *NetworkPolicyPeer) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + if len(m.LabelIdentities) > 0 { + for _, e := range m.LabelIdentities { + n += 1 + sovGenerated(uint64(e)) + } + } return n } @@ -4508,6 +4521,7 @@ func (this *NetworkPolicyPeer) String() string { `IPBlocks:` + repeatedStringForIPBlocks + `,`, `FQDNs:` + fmt.Sprintf("%v", this.FQDNs) + `,`, `ToServices:` + repeatedStringForToServices + `,`, + `LabelIdentities:` + fmt.Sprintf("%v", this.LabelIdentities) + `,`, `}`, }, "") return s @@ -8205,6 +8219,82 @@ func (m *NetworkPolicyPeer) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.LabelIdentities = append(m.LabelIdentities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.LabelIdentities) == 0 { + m.LabelIdentities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.LabelIdentities = append(m.LabelIdentities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field LabelIdentities", wireType) + } default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/controlplane/v1beta2/generated.proto b/pkg/apis/controlplane/v1beta2/generated.proto index 658814a8d64..9d7b0fc2bf3 100644 --- a/pkg/apis/controlplane/v1beta2/generated.proto +++ b/pkg/apis/controlplane/v1beta2/generated.proto @@ -282,6 +282,10 @@ message NetworkPolicyPeer { // A list of ServiceReference. // This field can only be possibly set for NetworkPolicyPeer of egress rules. repeated ServiceReference toServices = 4; + + // A list of labelIdentities selected as ingress peers for stretched policy. + // This field can only be possibly set for NetworkPolicyPeer of ingress rules. + repeated uint32 labelIdentities = 5; } message NetworkPolicyReference { diff --git a/pkg/apis/controlplane/v1beta2/types.go b/pkg/apis/controlplane/v1beta2/types.go index 9147e2c90d3..b819c950d8d 100644 --- a/pkg/apis/controlplane/v1beta2/types.go +++ b/pkg/apis/controlplane/v1beta2/types.go @@ -313,6 +313,9 @@ type NetworkPolicyPeer struct { // A list of ServiceReference. // This field can only be possibly set for NetworkPolicyPeer of egress rules. ToServices []ServiceReference `json:"toServices,omitempty" protobuf:"bytes,4,rep,name=toServices"` + // A list of labelIdentities selected as ingress peers for stretched policy. + // This field can only be possibly set for NetworkPolicyPeer of ingress rules. + LabelIdentities []uint32 `json:"labelIdentities,omitempty" protobuf:"bytes,5,rep,name=labelIdentities"` } // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24"). The except entry describes CIDRs that should diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go index 1129b3c6c94..ba884352a93 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go @@ -1312,6 +1312,7 @@ func autoConvert_v1beta2_NetworkPolicyPeer_To_controlplane_NetworkPolicyPeer(in out.IPBlocks = *(*[]controlplane.IPBlock)(unsafe.Pointer(&in.IPBlocks)) out.FQDNs = *(*[]string)(unsafe.Pointer(&in.FQDNs)) out.ToServices = *(*[]controlplane.ServiceReference)(unsafe.Pointer(&in.ToServices)) + out.LabelIdentities = *(*[]uint32)(unsafe.Pointer(&in.LabelIdentities)) return nil } @@ -1325,6 +1326,7 @@ func autoConvert_controlplane_NetworkPolicyPeer_To_v1beta2_NetworkPolicyPeer(in out.IPBlocks = *(*[]IPBlock)(unsafe.Pointer(&in.IPBlocks)) out.FQDNs = *(*[]string)(unsafe.Pointer(&in.FQDNs)) out.ToServices = *(*[]ServiceReference)(unsafe.Pointer(&in.ToServices)) + out.LabelIdentities = *(*[]uint32)(unsafe.Pointer(&in.LabelIdentities)) return nil } diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go index 010e6b81c21..2a1be887f66 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go @@ -781,6 +781,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = make([]ServiceReference, len(*in)) copy(*out, *in) } + if in.LabelIdentities != nil { + in, out := &in.LabelIdentities, &out.LabelIdentities + *out = make([]uint32, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/controlplane/zz_generated.deepcopy.go b/pkg/apis/controlplane/zz_generated.deepcopy.go index 499379d82c0..5327d4f827d 100644 --- a/pkg/apis/controlplane/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/zz_generated.deepcopy.go @@ -781,6 +781,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = make([]ServiceReference, len(*in)) copy(*out, *in) } + if in.LabelIdentities != nil { + in, out := &in.LabelIdentities, &out.LabelIdentities + *out = make([]uint32, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index bad9d6c0b81..012d8199778 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -479,6 +479,11 @@ type NetworkPolicyPeer struct { // A NodeSelector cannot be set in AppliedTo field or set with any other selector. // +optional NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` + // Define scope of the Pod/NamespaceSelector(s) of this peer. + // Can only be used in ingress NetworkPolicyPeers. + // Defaults to "Cluster". + // +optional + Scope PeerScope `json:"scope,omitempty"` } // AppliedTo describes the grouping selector of workloads in AppliedTo field. @@ -533,6 +538,13 @@ const ( NamespaceMatchSelf NamespaceMatchType = "Self" ) +type PeerScope string + +const ( + ScopeCluster PeerScope = "Cluster" + ScopeClusterSet PeerScope = "ClusterSet" +) + // IPBlock describes a particular CIDR (Ex. "192.168.1.1/24") that is allowed // or denied to/from the workloads matched by a Spec.AppliedTo. type IPBlock struct { diff --git a/pkg/apiserver/handlers/featuregates/handler.go b/pkg/apiserver/handlers/featuregates/handler.go index b091b4238f6..20daa93e7a4 100644 --- a/pkg/apiserver/handlers/featuregates/handler.go +++ b/pkg/apiserver/handlers/featuregates/handler.go @@ -29,7 +29,7 @@ import ( "antrea.io/antrea/pkg/util/env" ) -var controllerGates = sets.NewString("Traceflow", "AntreaPolicy", "Egress", "NetworkPolicyStats", "NodeIPAM", "ServiceExternalIP") +var controllerGates = sets.NewString("Traceflow", "AntreaPolicy", "Egress", "NetworkPolicyStats", "NodeIPAM", "ServiceExternalIP", "Multicluster") var agentGates = sets.NewString("AntreaPolicy", "AntreaProxy", "Egress", "EndpointSlice", "Traceflow", "FlowExporter", "NetworkPolicyStats", "NodePortLocal", "AntreaIPAM", "Multicast", "ServiceExternalIP", "Multicluster") diff --git a/pkg/apiserver/handlers/featuregates/handler_test.go b/pkg/apiserver/handlers/featuregates/handler_test.go index 7670579e09b..0d2b440ae22 100644 --- a/pkg/apiserver/handlers/featuregates/handler_test.go +++ b/pkg/apiserver/handlers/featuregates/handler_test.go @@ -185,6 +185,7 @@ func Test_getControllerGatesResponse(t *testing.T) { {Component: "controller", Name: "NetworkPolicyStats", Status: "Enabled", Version: "BETA"}, {Component: "controller", Name: "NodeIPAM", Status: "Disabled", Version: "ALPHA"}, {Component: "controller", Name: "ServiceExternalIP", Status: "Disabled", Version: "ALPHA"}, + {Component: "controller", Name: "Multicluster", Status: "Disabled", Version: "ALPHA"}, }, }, } diff --git a/pkg/apiserver/openapi/zz_generated.openapi.go b/pkg/apiserver/openapi/zz_generated.openapi.go index 432bb670b36..8d0d69e32d6 100644 --- a/pkg/apiserver/openapi/zz_generated.openapi.go +++ b/pkg/apiserver/openapi/zz_generated.openapi.go @@ -1532,6 +1532,21 @@ func schema_pkg_apis_controlplane_v1beta2_NetworkPolicyPeer(ref common.Reference }, }, }, + "labelIdentities": { + SchemaProps: spec.SchemaProps{ + Description: "A list of labelIdentities selected as ingress peers for stretched policy. This field can only be possibly set for NetworkPolicyPeer of ingress rules.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: 0, + Type: []string{"integer"}, + Format: "int64", + }, + }, + }, + }, + }, }, }, }, diff --git a/pkg/config/controller/config.go b/pkg/config/controller/config.go index 22a51f58fa7..5e1387fdfc5 100644 --- a/pkg/config/controller/config.go +++ b/pkg/config/controller/config.go @@ -67,6 +67,17 @@ type ControllerConfig struct { NodeIPAM NodeIPAMConfig `yaml:"nodeIPAM"` // IPsec CSR signer configuration IPsecCSRSignerConfig IPsecCSRSignerConfig `yaml:"ipsecCSRSigner"` + // Multicluster configuration options. + Multicluster MulticlusterConfig `yaml:"multicluster,omitempty"` +} + +type MulticlusterConfig struct { + // Enable Multicluster which allow Antrea-native policies to select peers + // from other clusters in a ClusterSet. + Enable bool `yaml:"enable,omitempty"` + // The Namespace where the Antrea Multi-cluster controller is running. + // The default is antrea-agent's Namespace. + Namespace string `yaml:"namespace,omitempty"` } type IPsecCSRSignerConfig struct { diff --git a/pkg/controller/labelidentity/controller.go b/pkg/controller/labelidentity/controller.go new file mode 100644 index 00000000000..3accad43e7a --- /dev/null +++ b/pkg/controller/labelidentity/controller.go @@ -0,0 +1,121 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labelidentity + +import ( + "sync/atomic" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + mcv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + mcinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions/multicluster/v1alpha1" +) + +const ( + controllerName = "LabelIdentityController" + // Set resyncPeriod to 0 to disable resyncing. + resyncPeriod time.Duration = 0 +) + +// eventsCounter is used to keep track of the number of occurrences of an event type. It uses the +// low-level atomic memory primitives from the sync/atomic package to provide atomic operations +// (Increment and Load). +// There is a known-bug on 32-bit architectures for sync/atomic: +// On ARM, 386, and 32-bit MIPS, it is the caller's responsibility to arrange for 64-bit alignment +// of 64-bit words accessed atomically. The first word in a variable or in an allocated struct, +// array, or slice can be relied upon to be 64-bit aligned. +// As a result, instances of eventsCounter should be allocated when using them in structs; they +// should not be embedded directly. +type eventsCounter struct { + count uint64 +} + +func (c *eventsCounter) Increment() { + atomic.AddUint64(&c.count, 1) +} + +func (c *eventsCounter) Load() uint64 { + return atomic.LoadUint64(&c.count) +} + +type Controller struct { + labelInformer mcinformers.LabelIdentityInformer + // labelListerSynced is a function which returns true if the LabelIdentity shared informer + // has been synced at least once + labelListerSynced cache.InformerSynced + // labelAddEvents tracks the number of LabelIdentity Add events that have been processed. + labelAddEvents *eventsCounter + // labelIdentityIndex is the stores the current state of the LabelIdentities and any selector + // that matches these LabelIdentities. + labelIdentityIndex *LabelIdentityIndex +} + +func NewLabelIdentityController(index *LabelIdentityIndex, + labelInformer mcinformers.LabelIdentityInformer) *Controller { + c := &Controller{ + labelIdentityIndex: index, + labelInformer: labelInformer, + labelListerSynced: labelInformer.Informer().HasSynced, + labelAddEvents: new(eventsCounter), + } + labelInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.addLabelIdentity, + // LabelIdentities are not expected to have update events after they are created. + // Update will be done by deleting existing one and recreate a new LabelIdentity with new ID. + UpdateFunc: nil, + DeleteFunc: c.deleteLabelIdentity, + }, + resyncPeriod, + ) + return c +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + klog.InfoS("Starting controller", "controller", controllerName) + defer klog.InfoS("Shutting down controller", "controller", controllerName) + + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.labelListerSynced) { + klog.Error("Failed to wait for label lister sync") + return + } + initialLabelCount := len(c.labelInformer.Informer().GetStore().List()) + // Wait until initial label identities are processed before setting labelIdentityIndex as synced. + if err := wait.PollImmediateUntil(100*time.Millisecond, func() (done bool, err error) { + if uint64(initialLabelCount) > c.labelAddEvents.Load() { + return false, nil + } + return true, nil + }, stopCh); err == nil { + c.labelIdentityIndex.setSynced(true) + } + <-stopCh +} + +func (c *Controller) addLabelIdentity(obj interface{}) { + labelIdentity := obj.(*mcv1alpha1.LabelIdentity) + klog.InfoS("Processing LabelIdentity ADD event", "label", labelIdentity.Spec.Label, "id", labelIdentity.Spec.ID) + c.labelIdentityIndex.AddLabelIdentity(labelIdentity.Spec.Label, labelIdentity.Spec.ID) + c.labelAddEvents.Increment() +} + +func (c *Controller) deleteLabelIdentity(obj interface{}) { + labelIdentity := obj.(*mcv1alpha1.LabelIdentity) + klog.InfoS("Processing LabelIdentity DELETE event", "label", labelIdentity.Spec.Label) + c.labelIdentityIndex.DeleteLabelIdentity(labelIdentity.Spec.Label) +} diff --git a/pkg/controller/labelidentity/controller_test.go b/pkg/controller/labelidentity/controller_test.go new file mode 100644 index 00000000000..bf091fc3ced --- /dev/null +++ b/pkg/controller/labelidentity/controller_test.go @@ -0,0 +1,84 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labelidentity + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + + mcv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + fakeversioned "antrea.io/antrea/multicluster/pkg/client/clientset/versioned/fake" + crdinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions" +) + +const informerDefaultResync = 30 * time.Second + +var ( + labelIdentityA = &mcv1alpha1.LabelIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterA-3de4f2hdf1", + }, + Spec: mcv1alpha1.LabelIdentitySpec{ + Label: labelA, + ID: 1, + }, + } + labelIdentityB = &mcv1alpha1.LabelIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterA-23few454fg", + }, + Spec: mcv1alpha1.LabelIdentitySpec{ + Label: labelB, + ID: 2, + }, + } +) + +func TestGroupEntityControllerRun(t *testing.T) { + // Even with only 1 buffer, the code should work as expected - as opposed to hanging somewhere. + originalEventChanSize := eventChanSize + eventChanSize = 1 + defer func() { + eventChanSize = originalEventChanSize + }() + + index := NewLabelIdentityIndex() + var objs []runtime.Object + initialLabelIdentities := []*mcv1alpha1.LabelIdentity{labelIdentityA, labelIdentityB} + for _, l := range initialLabelIdentities { + objs = append(objs, l) + } + mcClient := fakeversioned.NewSimpleClientset(objs...) + mcInformerFactory := crdinformers.NewSharedInformerFactory(mcClient, informerDefaultResync) + stopCh := make(chan struct{}) + + c := NewLabelIdentityController(index, mcInformerFactory.Multicluster().V1alpha1().LabelIdentities()) + assert.False(t, index.HasSynced(), "LabelIdentityIndex has been synced before starting InformerFactories") + + mcInformerFactory.Start(stopCh) + assert.False(t, index.HasSynced(), "LabelIdentityIndex has been synced before starting LabelIdentityController") + + go c.labelIdentityIndex.Run(stopCh) + go c.Run(stopCh) + + assert.NoError(t, wait.Poll(10*time.Millisecond, time.Second, func() (done bool, err error) { + return index.HasSynced(), nil + }), "LabelIdentityIndex hasn't been synced in 1 second after starting LabelIdentityController") +} diff --git a/pkg/controller/labelidentity/label_group_index.go b/pkg/controller/labelidentity/label_group_index.go new file mode 100644 index 00000000000..425d77e5bdd --- /dev/null +++ b/pkg/controller/labelidentity/label_group_index.go @@ -0,0 +1,508 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labelidentity + +import ( + "fmt" + "regexp" + "strings" + "sync" + "sync/atomic" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/controller/types" +) + +const ( + // Cluster scoped selectors are stored under empty Namespace in indice. + emptyNamespace = "" + policyIndex = "policyIndex" +) + +var ( + // eventChanSize is declared as a variable to allow overriding for testing. + eventChanSize = 1000 + // labelRegex knows how to decompose a normalized label identity. + labelRegex = regexp.MustCompile(`ns:(?P(.)*)&pod:(?P(.)*)`) + nsIndex = labelRegex.SubexpIndex("nslabels") + podIndex = labelRegex.SubexpIndex("podlabels") +) + +// eventHandler is the registered callback for policy re-sync +type eventHandler func(policyKey string) + +type Interface interface { + // AddSelector adds or updates a selectorItem when a new selector is added to a policy. + AddSelector(selector *types.GroupSelector, policyKey string) []uint32 + // DeleteSelector deletes or updates a selectorItem when a selector is deleted from a policy. + DeleteSelector(selectorKey string, policyKey string) + // GetLabelIdentityIDs retrieves the label identity IDs selected by the provided selectorItem keys. + GetLabelIdentityIDs(selectorKey string) []uint32 + // SetPolicySelectors registers a policy's selectors with the index. + SetPolicySelectors(selectors []*types.GroupSelector, policyKey string) []uint32 + // DeletePolicySelectors removes any selectors from referring to the policy being deleted. + DeletePolicySelectors(policyKey string) + // AddLabelIdentity adds LabelIdentity-ID mapping to the index. + AddLabelIdentity(labelKey string, id uint32) + // DeleteLabelIdentity deletes a LabelIdentity from the index. + DeleteLabelIdentity(labelKey string) + // AddEventHandler registers an eventHandler with the index. + AddEventHandler(handler eventHandler) + // Run starts the index. + Run(stopCh <-chan struct{}) + // HasSynced returns true if the interface has been initialized with the full lists of LabelIdentities. + HasSynced() bool +} + +type selectorItemUpdateEvent string + +const ( + selectorMatchedLabelAdd selectorItemUpdateEvent = "labelAdd" + selectorMatchedLabelDelete selectorItemUpdateEvent = "labelDelete" + selectorMatchedPolicyAdd selectorItemUpdateEvent = "policyAdd" + selectorMatchedPolicyDelete selectorItemUpdateEvent = "policyDelete" +) + +// selectorItem represents a ClusterSet-scope selector from Antrea-native policies. +// It also stores the LabelIdentity keys that this selector currently selects, as well +// as the keys of Antrea-native policies that have this selector. +type selectorItem struct { + selector *types.GroupSelector + // Keys are the normalized labels of matching LabelIdentities + labelIdentityKeys sets.String + // Keys are the UIDs of the policies that have the selector in their specs. + policyKeys sets.String +} + +func (s *selectorItem) getKey() string { + return s.selector.NormalizedName +} + +// labelIdentityMatch is constructed from a LabelIdentity and used for matching +// between LabelIdentity and selectorItems. It also stores the current selectorItems +// that matches this LabelIdentity. +type labelIdentityMatch struct { + id uint32 + namespace string + namespaceLabels map[string]string + podLabels map[string]string + selectorItemKeys sets.String +} + +// matches knows if a LabelIdentity matches a selectorItem. +func (l *labelIdentityMatch) matches(s *selectorItem) bool { + selectorItemNamespace := s.selector.Namespace + if selectorItemNamespace != emptyNamespace && selectorItemNamespace != l.namespace { + return false + } + if s.selector.NamespaceSelector != nil && !s.selector.NamespaceSelector.Matches(labels.Set(l.namespaceLabels)) { + return false + } + // At this stage Namespace has matched + if s.selector.PodSelector != nil { + return s.selector.PodSelector.Matches(labels.Set(l.podLabels)) + } + // SelectorItem selects all when all selectors are missing. + return true +} + +// constructMapFromLabelString parses label string of format "app=client,env=dev" into a map. +func constructMapFromLabelString(s string) map[string]string { + m := map[string]string{} + kvs := strings.Split(s, ",") + for _, kv := range kvs { + kvpair := strings.Split(kv, "=") + m[kvpair[0]] = kvpair[1] + } + return m +} + +// newLabelIdentityMatch constructs a labelIdentityMatch from a normalized LabelIdentity string. +func newLabelIdentityMatch(labelIdentity string, id uint32) *labelIdentityMatch { + labelMatches := labelRegex.FindStringSubmatch(labelIdentity) + nsLabels := constructMapFromLabelString(labelMatches[nsIndex]) + podLabels := constructMapFromLabelString(labelMatches[podIndex]) + + namespace := nsLabels[apiv1.LabelMetadataName] + return &labelIdentityMatch{ + id: id, + namespace: namespace, + namespaceLabels: nsLabels, + podLabels: podLabels, + selectorItemKeys: sets.NewString(), + } +} + +// selectorItemKeyFunc knows how to get the key of a selectorItem. +func selectorItemKeyFunc(obj interface{}) (string, error) { + sItem, ok := obj.(*selectorItem) + if !ok { + return "", fmt.Errorf("object is not of type *selectorItem: %v", obj) + } + return sItem.getKey(), nil +} + +func newSelectorItemStore() cache.Indexer { + indexers := cache.Indexers{ + cache.NamespaceIndex: func(obj interface{}) ([]string, error) { + sItem, ok := obj.(*selectorItem) + if !ok { + return []string{}, nil + } + // sItem.Selector.Namespace == "" means it's a cluster scoped selector, we index it as it is. + return []string{sItem.selector.Namespace}, nil + }, + policyIndex: func(obj interface{}) ([]string, error) { + sItem, ok := obj.(*selectorItem) + if !ok { + return []string{}, nil + } + return sItem.policyKeys.UnsortedList(), nil + }, + } + return cache.NewIndexer(selectorItemKeyFunc, indexers) +} + +// LabelIdentityIndex implements Interface. +type LabelIdentityIndex struct { + lock sync.RWMutex + // labelIdentities stores all labelIdentityMatches, with the normalized labels of LabelIdentity as map key. + labelIdentities map[string]*labelIdentityMatch + // labelIdentityNamespaceIndex is an index from Namespace to LabelIdentity keys in that Namespace. + labelIdentityNamespaceIndex map[string]sets.String + // selectorItems stores all selectorItems, indexed by Namespace and policy keys. + selectorItems cache.Indexer + + eventChan chan string + // eventHandlers is a list of callbacks registered for policies to be re-processed due to + // LabelIdentity events. + eventHandlers []eventHandler + + // synced stores a boolean value, which tracks if the LabelIdentityIndex has been initialized with + // the full lists of LabelIdentities. + synced *atomic.Value +} + +func NewLabelIdentityIndex() *LabelIdentityIndex { + synced := &atomic.Value{} + synced.Store(false) + index := &LabelIdentityIndex{ + labelIdentities: map[string]*labelIdentityMatch{}, + labelIdentityNamespaceIndex: map[string]sets.String{}, + selectorItems: newSelectorItemStore(), + eventChan: make(chan string, eventChanSize), + eventHandlers: []eventHandler{}, + synced: synced, + } + return index +} + +func (i *LabelIdentityIndex) updateSelectorItem(sItem *selectorItem, updateType selectorItemUpdateEvent, updateKey string) { + // Make a copy of selectorItem's fields as modifying the original object affects indexing. + labelIdentities, policies := sItem.labelIdentityKeys.Union(nil), sItem.policyKeys.Union(nil) + switch updateType { + case selectorMatchedLabelAdd: + labelIdentities.Insert(updateKey) + case selectorMatchedLabelDelete: + labelIdentities.Delete(updateKey) + case selectorMatchedPolicyAdd: + policies.Insert(updateKey) + case selectorMatchedPolicyDelete: + policies.Delete(updateKey) + } + // Construct a new selectorItem since objects got from ThreadSafeStore should be + // read-only. Indexers will break otherwise. + newSelectorItem := &selectorItem{ + selector: sItem.selector, + labelIdentityKeys: labelIdentities, + policyKeys: policies, + } + i.selectorItems.Update(newSelectorItem) +} + +// AddSelector registers a selectorItem to policy mapping with the LabelIdentityIndex, +// and returns the list of LabelIdentity IDs that the selector selects. +func (i *LabelIdentityIndex) AddSelector(selector *types.GroupSelector, policyKey string) []uint32 { + i.lock.Lock() + defer i.lock.Unlock() + + selectorKey, selectorNS := selector.NormalizedName, selector.Namespace + if s, exists, _ := i.selectorItems.GetByKey(selectorKey); exists { + sItem := s.(*selectorItem) + i.updateSelectorItem(sItem, selectorMatchedPolicyAdd, policyKey) + return i.getMatchedLabelIdentityIDs(sItem) + } + sItem := &selectorItem{ + selector: selector, + labelIdentityKeys: sets.NewString(), + policyKeys: sets.NewString(policyKey), + } + i.selectorItems.Add(sItem) + if selectorNS != emptyNamespace { + // Scan for LabelIdentity matches in a specific Namespace. + // Note that in multicluster context, the "Namespace sameness" concept applies, which means that + // Namespaces with the same name are considered to be the same Namespace across the ClusterSet. + // For more information, refer to + // https://github.com/kubernetes/community/blob/master/sig-multicluster/namespace-sameness-position-statement.md + labelIdentityKeys := i.labelIdentityNamespaceIndex[selectorNS] + i.scanLabelIdentityMatches(labelIdentityKeys, sItem) + } else { + // Scan for LabelIdentity matches globally. + for _, labelIdentityKeys := range i.labelIdentityNamespaceIndex { + i.scanLabelIdentityMatches(labelIdentityKeys, sItem) + } + } + return i.getMatchedLabelIdentityIDs(sItem) +} + +// DeleteSelector removes a selectorItem from referring to the policy being deleted. +func (i *LabelIdentityIndex) DeleteSelector(selectorKey string, policyKey string) { + i.lock.Lock() + defer i.lock.Unlock() + + i.deleteSelector(selectorKey, policyKey) +} + +func (i *LabelIdentityIndex) deleteSelector(selectorKey string, policyKey string) { + s, exists, _ := i.selectorItems.GetByKey(selectorKey) + if !exists { + return + } + sItem := s.(*selectorItem) + if sItem.policyKeys.Equal(sets.NewString(policyKey)) { + // delete the selectorItem and any LabelIdentity mappings if there's no + // policy left that has the selector anymore. + for lkey := range sItem.labelIdentityKeys { + labelIdentity := i.labelIdentities[lkey] + labelIdentity.selectorItemKeys.Delete(selectorKey) + } + i.selectorItems.Delete(sItem) + } else { + i.updateSelectorItem(sItem, selectorMatchedPolicyDelete, policyKey) + } +} + +// SetPolicySelectors registers ClusterSet-scope policy selectors with the labelIdentityIndex, +// and then retrieves all the LabelIdentity IDs that currently match these selectors. +func (i *LabelIdentityIndex) SetPolicySelectors(selectors []*types.GroupSelector, policyKey string) []uint32 { + var labelIdentityIDs []uint32 + newSelectors := map[string]*types.GroupSelector{} + for _, s := range selectors { + klog.V(4).InfoS("Getting matched LabelIdentity for policy selector", "selector", s.NormalizedName, "policy", policyKey) + newSelectors[s.NormalizedName] = s + } + originalSelectors := i.getPolicySelectors(policyKey) + for selKey, sel := range newSelectors { + if _, exists := originalSelectors[selKey]; exists { + // These clusterset-scoped selectors are already bound to the policy in labelIdentityIndex. + // We can simply read matched label identity IDs from the index. + selectedLabelIDs := i.GetLabelIdentityIDs(selKey) + labelIdentityIDs = append(labelIdentityIDs, selectedLabelIDs...) + // Remove matched clusterset-scoped selectors of the policy before and after the update. + // The selectors remaining in originalSelectors will need to be removed from the policy. + delete(originalSelectors, selKey) + } else { + selectedLabelIDs := i.AddSelector(sel, policyKey) + labelIdentityIDs = append(labelIdentityIDs, selectedLabelIDs...) + } + } + // The policy no longer has these selectors. + for selectorKey := range originalSelectors { + i.DeleteSelector(selectorKey, policyKey) + } + // Dedup label identity IDs in-place. + seen := map[uint32]struct{}{} + idx := 0 + for _, id := range labelIdentityIDs { + if _, exists := seen[id]; !exists { + seen[id] = struct{}{} + labelIdentityIDs[idx] = id + idx++ + } + } + return labelIdentityIDs[:idx] +} + +func (i *LabelIdentityIndex) getPolicySelectors(policyKey string) map[string]*types.GroupSelector { + i.lock.RLock() + defer i.lock.RUnlock() + + res := map[string]*types.GroupSelector{} + selectors, _ := i.selectorItems.ByIndex(policyIndex, policyKey) + for _, s := range selectors { + sel := s.(*selectorItem) + res[sel.getKey()] = sel.selector + } + return res +} + +func (i *LabelIdentityIndex) DeletePolicySelectors(policyKey string) { + i.lock.Lock() + defer i.lock.Unlock() + + selectors, _ := i.selectorItems.ByIndex(policyIndex, policyKey) + for _, s := range selectors { + sel := s.(*selectorItem) + i.deleteSelector(sel.getKey(), policyKey) + } +} + +func (i *LabelIdentityIndex) GetLabelIdentityIDs(selectorKey string) []uint32 { + i.lock.RLock() + defer i.lock.RUnlock() + + if s, exists, _ := i.selectorItems.GetByKey(selectorKey); exists { + sel := s.(*selectorItem) + return i.getMatchedLabelIdentityIDs(sel) + } + return []uint32{} +} + +func (i *LabelIdentityIndex) getMatchedLabelIdentityIDs(sItem *selectorItem) []uint32 { + var ids []uint32 + for lKey := range sItem.labelIdentityKeys { + labelIdentity := i.labelIdentities[lKey] + ids = append(ids, labelIdentity.id) + } + return ids +} + +func (i *LabelIdentityIndex) scanLabelIdentityMatches(labelIdentityKeys sets.String, sItem *selectorItem) { + for lkey := range labelIdentityKeys { + labelIdentity := i.labelIdentities[lkey] + if labelIdentity.matches(sItem) { + sItem.labelIdentityKeys.Insert(lkey) + labelIdentity.selectorItemKeys.Insert(sItem.getKey()) + } + } +} + +func (i *LabelIdentityIndex) AddLabelIdentity(labelKey string, id uint32) { + i.lock.Lock() + defer i.lock.Unlock() + + existingLabelMatch, exists := i.labelIdentities[labelKey] + if exists { + if existingLabelMatch.id != id { + existingLabelMatch.id = id + i.notifyPoliciesForLabelIdentityUpdate(existingLabelMatch) + } + return + } + klog.V(2).InfoS("Adding new LabelIdentity", "label", labelKey) + labelIdentityMatch := newLabelIdentityMatch(labelKey, id) + i.labelIdentities[labelKey] = labelIdentityMatch + if keys, ok := i.labelIdentityNamespaceIndex[labelIdentityMatch.namespace]; ok { + keys.Insert(labelKey) + } else { + i.labelIdentityNamespaceIndex[labelIdentityMatch.namespace] = sets.NewString(labelKey) + } + i.scanSelectorItemMatches(labelIdentityMatch, labelKey) +} + +func (i *LabelIdentityIndex) DeleteLabelIdentity(labelKey string) { + i.lock.Lock() + defer i.lock.Unlock() + + l, exists := i.labelIdentities[labelKey] + if !exists { + klog.V(2).InfoS("LabelIdentity is already deleted from the index", "label", labelKey) + return + } + klog.V(2).InfoS("Deleting LabelIdentity", "label", labelKey) + labelIdentityNamespace := l.namespace + policyKeysToNotify := sets.NewString() + for sKey := range l.selectorItemKeys { + if s, exists, _ := i.selectorItems.GetByKey(sKey); exists { + sItem := s.(*selectorItem) + policyKeysToNotify = policyKeysToNotify.Union(sItem.policyKeys) + i.updateSelectorItem(sItem, selectorMatchedLabelDelete, labelKey) + } + } + delete(i.labelIdentities, labelKey) + if labelKeys, ok := i.labelIdentityNamespaceIndex[labelIdentityNamespace]; ok { + labelKeys.Delete(labelKey) + if len(labelKeys) == 0 { + // There are no more labelIdentities in that Namespace + delete(i.labelIdentityNamespaceIndex, labelIdentityNamespace) + } + } + i.notify(policyKeysToNotify) +} + +func (i *LabelIdentityIndex) notify(policyKeys sets.String) { + for k := range policyKeys { + klog.V(2).InfoS("Adding policy to the resync chan", "policyKey", k) + i.eventChan <- k + } +} + +func (i *LabelIdentityIndex) notifyPoliciesForLabelIdentityUpdate(l *labelIdentityMatch) { + for sKey := range l.selectorItemKeys { + if s, exists, _ := i.selectorItems.GetByKey(sKey); exists { + sItem := s.(*selectorItem) + i.notify(sItem.policyKeys) + } + } +} + +// scanSelectorItemMatches scans all selectorItems that can possibly match the LabelIdentity. +// If there are new matches, all policies that possess the selectorItem will be notified as +// a new LabelIdentity ID will be matched for that policy. +func (i *LabelIdentityIndex) scanSelectorItemMatches(l *labelIdentityMatch, normalizedLabel string) { + nsSelectors, _ := i.selectorItems.ByIndex(cache.NamespaceIndex, l.namespace) + clusterSelectors, _ := i.selectorItems.ByIndex(cache.NamespaceIndex, emptyNamespace) + allSelectors := append(nsSelectors, clusterSelectors...) + for _, n := range allSelectors { + sItem := n.(*selectorItem) + if l.matches(sItem) { + l.selectorItemKeys.Insert(sItem.getKey()) + i.updateSelectorItem(sItem, selectorMatchedLabelAdd, normalizedLabel) + i.notify(sItem.policyKeys) + } + } +} + +func (i *LabelIdentityIndex) AddEventHandler(handler eventHandler) { + i.eventHandlers = append(i.eventHandlers, handler) +} + +func (i *LabelIdentityIndex) HasSynced() bool { + return i.synced.Load().(bool) +} + +func (i *LabelIdentityIndex) setSynced(synced bool) { + i.synced.Store(synced) +} + +func (i *LabelIdentityIndex) Run(stopCh <-chan struct{}) { + klog.Info("Starting LabelIdentityIndex") + for { + select { + case <-stopCh: + klog.Info("Stopping LabelIdentityIndex") + return + case policyKey := <-i.eventChan: + for _, handler := range i.eventHandlers { + handler(policyKey) + } + } + } +} diff --git a/pkg/controller/labelidentity/label_group_index_test.go b/pkg/controller/labelidentity/label_group_index_test.go new file mode 100644 index 00000000000..f2f4c14e788 --- /dev/null +++ b/pkg/controller/labelidentity/label_group_index_test.go @@ -0,0 +1,547 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labelidentity + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + "antrea.io/antrea/pkg/controller/types" +) + +var ( + pSelWeb = &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "web"}, + } + pSelDB = &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "db"}, + } + nsSelTest = &metav1.LabelSelector{ + MatchLabels: map[string]string{"purpose": "test"}, + } + selectorA = types.NewGroupSelector("", pSelWeb, nil, nil, nil) + selectorB = types.NewGroupSelector("", pSelWeb, nsSelTest, nil, nil) + selectorC = types.NewGroupSelector("", pSelDB, nil, nil, nil) + selectorD = types.NewGroupSelector("testing", pSelDB, nil, nil, nil) + selectorE = types.NewGroupSelector("random", pSelDB, nil, nil, nil) + selectorItemA = &selectorItem{ + selector: selectorA, + } + selectorItemB = &selectorItem{ + selector: selectorB, + } + selectorItemC = &selectorItem{ + selector: selectorC, + } + selectorItemD = &selectorItem{ + selector: selectorD, + } + selectorItemE = &selectorItem{ + selector: selectorE, + } + labelA = "ns:kubernetes.io/metadata.name=testing,purpose=test&pod:app=web" + labelB = "ns:kubernetes.io/metadata.name=testing,purpose=test&pod:app=db" + labelC = "ns:kubernetes.io/metadata.name=nomatch,purpose=nomatch&pod:app=db" +) + +func TestLabelIdentityMatch(t *testing.T) { + tests := []struct { + label string + selector *selectorItem + expectMatch bool + }{ + { + label: labelA, + selector: selectorItemA, + expectMatch: true, + }, + { + label: labelA, + selector: selectorItemB, + expectMatch: true, + }, + { + label: labelA, + selector: selectorItemC, + expectMatch: false, + }, + { + label: labelA, + selector: selectorItemD, + expectMatch: false, + }, + { + label: labelB, + selector: selectorItemB, + expectMatch: false, + }, + { + label: labelB, + selector: selectorItemC, + expectMatch: true, + }, + { + label: labelB, + selector: selectorItemD, + expectMatch: true, + }, + { + label: labelB, + selector: selectorItemE, + expectMatch: false, + }, + { + label: labelC, + selector: selectorItemB, + expectMatch: false, + }, + { + label: labelC, + selector: selectorItemC, + expectMatch: true, + }, + { + label: labelC, + selector: selectorItemD, + expectMatch: false, + }, + { + label: labelC, + selector: selectorItemE, + expectMatch: false, + }, + } + for _, tt := range tests { + labelMatch := newLabelIdentityMatch(tt.label, 1) + matched := labelMatch.matches(tt.selector) + assert.Equalf(t, tt.expectMatch, matched, "Unexpected matching status for %s and %s.", tt.label, tt.selector.getKey()) + } +} + +func TestAddSelector(t *testing.T) { + tests := []struct { + name string + selectorToAdd *types.GroupSelector + existingPolicyKey string + policyKey string + expMatchedIDs []uint32 + expLabelIdentityKeys sets.String + expPolicyKeys sets.String + }{ + { + name: "cluster-wide app=web", + selectorToAdd: types.NewGroupSelector("", pSelWeb, nil, nil, nil), + policyKey: "policyA", + expMatchedIDs: []uint32{1}, + expLabelIdentityKeys: sets.NewString(labelA), + expPolicyKeys: sets.NewString("policyA"), + }, + { + name: "cluster-wide app=web another policy", + selectorToAdd: types.NewGroupSelector("", pSelWeb, nil, nil, nil), + existingPolicyKey: "policyA", + policyKey: "policyB", + expMatchedIDs: []uint32{1}, + expLabelIdentityKeys: sets.NewString(labelA), + expPolicyKeys: sets.NewString("policyA", "policyB"), + }, + { + name: "pod app=web and ns purpose=test", + selectorToAdd: types.NewGroupSelector("", pSelWeb, nsSelTest, nil, nil), + policyKey: "policyB", + expMatchedIDs: []uint32{1}, + expLabelIdentityKeys: sets.NewString(labelA), + expPolicyKeys: sets.NewString("policyB"), + }, + { + name: "cluster-wide app=db", + selectorToAdd: types.NewGroupSelector("", pSelDB, nil, nil, nil), + policyKey: "policyC", + expMatchedIDs: []uint32{2, 3}, + expLabelIdentityKeys: sets.NewString(labelB, labelC), + expPolicyKeys: sets.NewString("policyC"), + }, + { + name: "app=db in ns testing", + selectorToAdd: types.NewGroupSelector("testing", pSelDB, nil, nil, nil), + policyKey: "policyD", + expMatchedIDs: []uint32{2}, + expLabelIdentityKeys: sets.NewString(labelB), + expPolicyKeys: sets.NewString("policyD"), + }, + { + name: "app=db in ns random", + selectorToAdd: types.NewGroupSelector("random", pSelDB, nil, nil, nil), + policyKey: "policyE", + expMatchedIDs: []uint32{}, + expLabelIdentityKeys: sets.NewString(), + expPolicyKeys: sets.NewString("policyE"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := NewLabelIdentityIndex() + i.AddLabelIdentity(labelA, 1) + i.AddLabelIdentity(labelB, 2) + i.AddLabelIdentity(labelC, 3) + if tt.existingPolicyKey != "" { + i.AddSelector(tt.selectorToAdd, tt.existingPolicyKey) + } + idsMatched := i.AddSelector(tt.selectorToAdd, tt.policyKey) + assert.ElementsMatch(t, tt.expMatchedIDs, idsMatched) + s, exists, _ := i.selectorItems.GetByKey(tt.selectorToAdd.NormalizedName) + require.True(t, exists, "Failed to add selector %s to the LabelIdentityIndex", tt.name) + sItem := s.(*selectorItem) + assert.Equalf(t, tt.expLabelIdentityKeys, sItem.labelIdentityKeys, "Unexpected label identity keys for selectorItem") + assert.Equalf(t, tt.expPolicyKeys, sItem.policyKeys, "Unexpected policy keys for selectorItem") + }) + } +} + +func TestDeletePolicySelectors(t *testing.T) { + tests := []struct { + policyKey string + staleSelector string + }{ + { + policyKey: "policyA", + staleSelector: selectorItemD.getKey(), + }, + { + policyKey: "policyB", + staleSelector: selectorItemC.getKey(), + }, + } + i := NewLabelIdentityIndex() + i.AddLabelIdentity(labelB, 2) + i.AddLabelIdentity(labelC, 3) + i.AddSelector(selectorItemD.selector, "policyA") + i.AddSelector(selectorItemC.selector, "policyB") + for _, tt := range tests { + i.DeletePolicySelectors(tt.policyKey) + for k, l := range i.labelIdentities { + if l.selectorItemKeys.Has(tt.staleSelector) { + t.Errorf("Stale selector %s is not deleted from labelMatch %s", tt.staleSelector, k) + } + } + if _, exists, _ := i.selectorItems.GetByKey(tt.staleSelector); exists { + t.Errorf("Stale selector %s is not deleted from selectorItem cache", tt.staleSelector) + } + } +} + +func TestSetPolicySelectors(t *testing.T) { + tests := []struct { + name string + selectors []*types.GroupSelector + policyKey string + prevSelAdded []*types.GroupSelector + prevPolicyAdded string + expIDs []uint32 + expSelectorItems map[string]selectorItem + }{ + { + name: "new selector for policyA", + selectors: []*types.GroupSelector{ + types.NewGroupSelector("", pSelWeb, nil, nil, nil), + }, + policyKey: "policyA", + expIDs: []uint32{1}, + expSelectorItems: map[string]selectorItem{ + selectorItemA.selector.NormalizedName: { + labelIdentityKeys: sets.NewString(labelA), + policyKeys: sets.NewString("policyA"), + }, + }, + }, + { + name: "updated selectors for policyA", + selectors: []*types.GroupSelector{ + types.NewGroupSelector("", pSelWeb, nsSelTest, nil, nil), + types.NewGroupSelector("", pSelDB, nil, nil, nil), + }, + policyKey: "policyA", + prevPolicyAdded: "policyA", + prevSelAdded: []*types.GroupSelector{ + types.NewGroupSelector("", pSelWeb, nil, nil, nil), + }, + expIDs: []uint32{1, 2, 3}, + expSelectorItems: map[string]selectorItem{ + selectorItemB.selector.NormalizedName: { + labelIdentityKeys: sets.NewString(labelA), + policyKeys: sets.NewString("policyA"), + }, + selectorItemC.selector.NormalizedName: { + labelIdentityKeys: sets.NewString(labelB, labelC), + policyKeys: sets.NewString("policyA"), + }, + }, + }, + { + name: "existing selector and new selector for policyB", + selectors: []*types.GroupSelector{ + types.NewGroupSelector("", pSelWeb, nil, nil, nil), + types.NewGroupSelector("", pSelWeb, nsSelTest, nil, nil), + }, + policyKey: "policyB", + prevPolicyAdded: "policyA", + prevSelAdded: []*types.GroupSelector{ + types.NewGroupSelector("", pSelWeb, nsSelTest, nil, nil), + types.NewGroupSelector("", pSelDB, nil, nil, nil), + }, + expIDs: []uint32{1}, + expSelectorItems: map[string]selectorItem{ + selectorItemA.selector.NormalizedName: { + labelIdentityKeys: sets.NewString(labelA), + policyKeys: sets.NewString("policyB"), + }, + selectorItemB.selector.NormalizedName: { + labelIdentityKeys: sets.NewString(labelA), + policyKeys: sets.NewString("policyA", "policyB"), + }, + selectorItemC.selector.NormalizedName: { + labelIdentityKeys: sets.NewString(labelB, labelC), + policyKeys: sets.NewString("policyA"), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := NewLabelIdentityIndex() + i.AddLabelIdentity(labelA, 1) + i.AddLabelIdentity(labelB, 2) + i.AddLabelIdentity(labelC, 3) + if tt.prevPolicyAdded != "" { + i.SetPolicySelectors(tt.prevSelAdded, tt.prevPolicyAdded) + } + matchedIDs := i.SetPolicySelectors(tt.selectors, tt.policyKey) + assert.ElementsMatch(t, tt.expIDs, matchedIDs) + assert.Equalf(t, len(tt.expSelectorItems), len(i.selectorItems.List()), "Unexpected number of cached selectorItems") + for selKey, expSelItem := range tt.expSelectorItems { + s, exists, _ := i.selectorItems.GetByKey(selKey) + if !exists { + t.Errorf("Selector %s is not added", selKey) + } + sItem := s.(*selectorItem) + assert.Truef(t, sItem.policyKeys.Equal(expSelItem.policyKeys), "Unexpected policy keys for selectorItem %s", selKey) + assert.Truef(t, sItem.labelIdentityKeys.Equal(expSelItem.labelIdentityKeys), "Unexpected labelIdentity keys for selectorItem %s", selKey) + } + }) + } +} + +func TestAddLabelIdentity(t *testing.T) { + labelIdentityAOriginalID := uint32(1) + tests := []struct { + name string + normalizedLabel string + id uint32 + originalID *uint32 + expPolicyCalled []string + expLabelIdenities map[string]*labelIdentityMatch + }{ + { + name: "Add label identity A", + normalizedLabel: labelA, + id: 1, + expPolicyCalled: []string{"policyA", "policyB"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelA: { + id: 1, + selectorItemKeys: sets.NewString(selectorItemA.getKey(), selectorItemB.getKey()), + }, + }, + }, + { + name: "Update label identity A", + normalizedLabel: labelA, + id: 4, + originalID: &labelIdentityAOriginalID, + expPolicyCalled: []string{"policyA", "policyB", "policyA", "policyB"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelA: { + id: 4, + selectorItemKeys: sets.NewString(selectorItemA.getKey(), selectorItemB.getKey()), + }, + }, + }, + { + name: "Add label identity B", + normalizedLabel: labelB, + id: 2, + expPolicyCalled: []string{"policyA", "policyB", "policyD"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelB: { + id: 2, + selectorItemKeys: sets.NewString(selectorItemC.getKey(), selectorItemD.getKey()), + }, + }, + }, + { + name: "Add label identity C", + normalizedLabel: labelC, + id: 3, + expPolicyCalled: []string{"policyA", "policyB"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelC: { + id: 3, + selectorItemKeys: sets.NewString(selectorItemC.getKey()), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := NewLabelIdentityIndex() + i.SetPolicySelectors([]*types.GroupSelector{selectorA, selectorC}, "policyA") + i.SetPolicySelectors([]*types.GroupSelector{selectorB, selectorC}, "policyB") + i.SetPolicySelectors([]*types.GroupSelector{selectorD}, "policyD") + i.SetPolicySelectors([]*types.GroupSelector{selectorE}, "policyE") + stopCh := make(chan struct{}) + defer close(stopCh) + + go i.Run(stopCh) + var lock sync.Mutex + var actualPoliciesCalled []string + i.AddEventHandler(func(policyKey string) { + lock.Lock() + defer lock.Unlock() + actualPoliciesCalled = append(actualPoliciesCalled, policyKey) + }) + + if tt.originalID != nil { + i.AddLabelIdentity(tt.normalizedLabel, *tt.originalID) + } + i.AddLabelIdentity(tt.normalizedLabel, tt.id) + // Wait for event handler to handle label add event + time.Sleep(10 * time.Millisecond) + lock.Lock() + defer lock.Unlock() + assert.ElementsMatchf(t, actualPoliciesCalled, tt.expPolicyCalled, "Unexpected policy sync calls") + for key, l := range tt.expLabelIdenities { + actLabelMatch := i.labelIdentities[key] + assert.Equalf(t, actLabelMatch.id, l.id, "Unexpected id cached for label") + if !actLabelMatch.selectorItemKeys.Equal(l.selectorItemKeys) { + t.Errorf("Unexpected matched selectorItems for label %s in step %s", tt.normalizedLabel, tt.name) + } + } + }) + } +} + +func TestDeleteLabelIdentity(t *testing.T) { + tests := []struct { + name string + labelToDelete string + expPolicyCalled []string + expLabelIdenities map[string]*labelIdentityMatch + }{ + { + name: "Delete label identity A", + labelToDelete: labelA, + expPolicyCalled: []string{"policyA", "policyB"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelB: { + id: 2, + selectorItemKeys: sets.NewString(selectorItemC.getKey(), selectorItemD.getKey()), + }, + labelC: { + id: 3, + selectorItemKeys: sets.NewString(selectorItemC.getKey()), + }, + }, + }, + { + name: "Delete label identity B", + labelToDelete: labelB, + expPolicyCalled: []string{"policyA", "policyB", "policyD"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelA: { + id: 1, + selectorItemKeys: sets.NewString(selectorItemA.getKey(), selectorItemB.getKey()), + }, + labelC: { + id: 3, + selectorItemKeys: sets.NewString(selectorItemC.getKey()), + }, + }, + }, + { + name: "Delete label identity C", + labelToDelete: labelC, + expPolicyCalled: []string{"policyA", "policyB"}, + expLabelIdenities: map[string]*labelIdentityMatch{ + labelA: { + id: 1, + selectorItemKeys: sets.NewString(selectorItemA.getKey(), selectorItemB.getKey()), + }, + labelB: { + id: 2, + selectorItemKeys: sets.NewString(selectorItemC.getKey(), selectorItemD.getKey()), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := NewLabelIdentityIndex() + i.SetPolicySelectors([]*types.GroupSelector{selectorA, selectorC}, "policyA") + i.SetPolicySelectors([]*types.GroupSelector{selectorB, selectorC}, "policyB") + i.SetPolicySelectors([]*types.GroupSelector{selectorD}, "policyD") + i.SetPolicySelectors([]*types.GroupSelector{selectorE}, "policyE") + stopCh := make(chan struct{}) + defer close(stopCh) + + go i.Run(stopCh) + var lock sync.Mutex + var actualPoliciesCalled []string + i.AddEventHandler(func(policyKey string) { + lock.Lock() + defer lock.Unlock() + actualPoliciesCalled = append(actualPoliciesCalled, policyKey) + }) + // Preload the index with label identities to be deleted + i.AddLabelIdentity(labelA, 1) + i.AddLabelIdentity(labelB, 2) + i.AddLabelIdentity(labelC, 3) + // Reset the actualPoliciesCalled slice. Otherwise, the list will be filled with extra items + // in the channel from preloading label identities. + time.Sleep(10 * time.Millisecond) + lock.Lock() + actualPoliciesCalled = []string{} + lock.Unlock() + + i.DeleteLabelIdentity(tt.labelToDelete) + time.Sleep(10 * time.Millisecond) + lock.Lock() + defer lock.Unlock() + assert.ElementsMatchf(t, actualPoliciesCalled, tt.expPolicyCalled, "Unexpected policy sync calls") + for key, l := range tt.expLabelIdenities { + actLabelMatch := i.labelIdentities[key] + assert.Equalf(t, actLabelMatch.id, l.id, "Unexpected id cached for label") + if !actLabelMatch.selectorItemKeys.Equal(l.selectorItemKeys) { + t.Errorf("Unexpected matched selectorItems for label %s in step %s", tt.labelToDelete, tt.name) + } + } + }) + } +} diff --git a/pkg/controller/networkpolicy/crd_utils.go b/pkg/controller/networkpolicy/crd_utils.go index 113c5ac82c1..66a711d847e 100644 --- a/pkg/controller/networkpolicy/crd_utils.go +++ b/pkg/controller/networkpolicy/crd_utils.go @@ -149,10 +149,13 @@ func (n *NetworkPolicyController) toAntreaPeerForCRD(peers []v1alpha1.NetworkPol } var ipBlocks []controlplane.IPBlock var fqdns []string + var clusterSetScopeSelectors []*antreatypes.GroupSelector for _, peer := range peers { - // A v1alpha1.NetworkPolicyPeer will either have an IPBlock or FQDNs or a - // podSelector and/or namespaceSelector set or a reference to the - // ClusterGroup. + // A v1alpha1.NetworkPolicyPeer will have exactly one of the following fields set: + // - podSelector and/or namespaceSelector (in-cluster scope or ClusterSet scope) + // - reference to a Group/ClusterGroup + // - IPBlocks + // - FQDNs if peer.IPBlock != nil { ipBlock, err := toAntreaIPBlockForCRD(peer.IPBlock) if err != nil { @@ -174,12 +177,18 @@ func (n *NetworkPolicyController) toAntreaPeerForCRD(peers []v1alpha1.NetworkPol } else if peer.NodeSelector != nil { addressGroup := n.createAddressGroup("", nil, nil, nil, peer.NodeSelector) addressGroups = append(addressGroups, addressGroup) + } else if peer.Scope == v1alpha1.ScopeClusterSet { + clusterSetScopeSelectors = append(clusterSetScopeSelectors, antreatypes.NewGroupSelector(np.GetNamespace(), peer.PodSelector, peer.NamespaceSelector, nil, nil)) } else { addressGroup := n.createAddressGroup(np.GetNamespace(), peer.PodSelector, peer.NamespaceSelector, peer.ExternalEntitySelector, nil) addressGroups = append(addressGroups, addressGroup) } } - return &controlplane.NetworkPolicyPeer{AddressGroups: getAddressGroupNames(addressGroups), IPBlocks: ipBlocks, FQDNs: fqdns}, addressGroups + var labelIdentities []uint32 + if n.multiclusterEnabled { + labelIdentities = n.labelIdentityInterface.SetPolicySelectors(clusterSetScopeSelectors, internalNetworkPolicyKeyFunc(np)) + } + return &controlplane.NetworkPolicyPeer{AddressGroups: getAddressGroupNames(addressGroups), IPBlocks: ipBlocks, FQDNs: fqdns, LabelIdentities: labelIdentities}, addressGroups } // toNamespacedPeerForCRD creates an Antrea controlplane NetworkPolicyPeer for crdv1alpha1 NetworkPolicyPeer diff --git a/pkg/controller/networkpolicy/crd_utils_test.go b/pkg/controller/networkpolicy/crd_utils_test.go index c82b260630d..51f7d88c629 100644 --- a/pkg/controller/networkpolicy/crd_utils_test.go +++ b/pkg/controller/networkpolicy/crd_utils_test.go @@ -22,11 +22,13 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + featuregatetesting "k8s.io/component-base/featuregate/testing" "antrea.io/antrea/pkg/apis/controlplane" crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" crdv1alpha3 "antrea.io/antrea/pkg/apis/crd/v1alpha3" antreatypes "antrea.io/antrea/pkg/controller/types" + "antrea.io/antrea/pkg/features" ) func TestToAntreaServicesForCRD(t *testing.T) { @@ -280,6 +282,7 @@ func TestToAntreaPeerForCRD(t *testing.T) { direction controlplane.Direction namedPortExists bool cgExists bool + clusterSetScope bool }{ { name: "pod-ns-selector-peer-ingress", @@ -422,16 +425,40 @@ func TestToAntreaPeerForCRD(t *testing.T) { }, direction: controlplane.DirectionOut, }, + { + name: "stretched-policy-peer", + inPeers: []crdv1alpha1.NetworkPolicyPeer{ + { + PodSelector: &selectorA, + Scope: crdv1alpha1.ScopeClusterSet, + }, + }, + outPeer: controlplane.NetworkPolicyPeer{ + LabelIdentities: []uint32{1}, + }, + direction: controlplane.DirectionIn, + clusterSetScope: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, npc := newController() npc.addClusterGroup(&cgA) npc.cgStore.Add(&cgA) + if tt.clusterSetScope { + defer featuregatetesting.SetFeatureGateDuringTest(t, features.DefaultFeatureGate, features.Multicluster, true)() + labelIdentityA := "ns:kubernetes.io/metadata.name=testing,purpose=test&pod:foo1=bar1" + labelIdentityB := "ns:kubernetes.io/metadata.name=testing,purpose=test&pod:foo2=bar2" + npc.labelIdentityInterface.AddLabelIdentity(labelIdentityA, 1) + npc.labelIdentityInterface.AddLabelIdentity(labelIdentityB, 2) + } actualPeer, _ := npc.toAntreaPeerForCRD(tt.inPeers, testCNPObj, tt.direction, tt.namedPortExists) if !reflect.DeepEqual(tt.outPeer.AddressGroups, actualPeer.AddressGroups) { t.Errorf("Unexpected AddressGroups in Antrea Peer conversion. Expected %v, got %v", tt.outPeer.AddressGroups, actualPeer.AddressGroups) } + if !reflect.DeepEqual(tt.outPeer.LabelIdentities, actualPeer.LabelIdentities) { + t.Errorf("Unexpected LabelIdentities in Antrea Peer conversion. Expected %v, got %v", tt.outPeer.LabelIdentities, actualPeer.LabelIdentities) + } if len(tt.outPeer.IPBlocks) != len(actualPeer.IPBlocks) { t.Errorf("Unexpected number of IPBlocks in Antrea Peer conversion. Expected %v, got %v", len(tt.outPeer.IPBlocks), len(actualPeer.IPBlocks)) } diff --git a/pkg/controller/networkpolicy/networkpolicy_controller.go b/pkg/controller/networkpolicy/networkpolicy_controller.go index 88f6db5bd25..f0072cd11c5 100644 --- a/pkg/controller/networkpolicy/networkpolicy_controller.go +++ b/pkg/controller/networkpolicy/networkpolicy_controller.go @@ -54,6 +54,7 @@ import ( seclisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" crdv1a3listers "antrea.io/antrea/pkg/client/listers/crd/v1alpha3" "antrea.io/antrea/pkg/controller/grouping" + "antrea.io/antrea/pkg/controller/labelidentity" "antrea.io/antrea/pkg/controller/metrics" "antrea.io/antrea/pkg/controller/networkpolicy/store" antreatypes "antrea.io/antrea/pkg/controller/types" @@ -229,13 +230,18 @@ type NetworkPolicyController struct { // synced. internalGroupQueue workqueue.RateLimitingInterface - // internalNetworkPolicyMutex protects the internalNetworkPolicyStore from - // concurrent access during updates to the internal NetworkPolicy object. + // internalNetworkPolicyMutex prevents concurrent processing of internal networkpolicies who refer + // to the same addressgroups/appliedtogroups. internalNetworkPolicyMutex sync.RWMutex groupingInterface grouping.Interface // Added as a member to the struct to allow injection for testing. groupingInterfaceSynced func() bool + + labelIdentityInterface labelidentity.Interface + // Enable Multicluster which allow Antrea-native policies to select peer + // from other clusters in a ClusterSet. + multiclusterEnabled bool // heartbeatCh is an internal channel for testing. It's used to know whether all tasks have been // processed, and to count executions of each function. heartbeatCh chan heartbeat @@ -370,6 +376,7 @@ var anpIndexers = cache.Indexers{ func NewNetworkPolicyController(kubeClient clientset.Interface, crdClient versioned.Interface, groupingInterface grouping.Interface, + labelIdentityInterface labelidentity.Interface, namespaceInformer coreinformers.NamespaceInformer, serviceInformer coreinformers.ServiceInformer, networkPolicyInformer networkinginformers.NetworkPolicyInformer, @@ -382,7 +389,8 @@ func NewNetworkPolicyController(kubeClient clientset.Interface, addressGroupStore storage.Interface, appliedToGroupStore storage.Interface, internalNetworkPolicyStore storage.Interface, - internalGroupStore storage.Interface) *NetworkPolicyController { + internalGroupStore storage.Interface, + multiclusterEnabled bool) *NetworkPolicyController { n := &NetworkPolicyController{ kubeClient: kubeClient, crdClient: crdClient, @@ -399,10 +407,13 @@ func NewNetworkPolicyController(kubeClient clientset.Interface, internalGroupQueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "internalGroup"), groupingInterface: groupingInterface, groupingInterfaceSynced: groupingInterface.HasSynced, + labelIdentityInterface: labelIdentityInterface, + multiclusterEnabled: multiclusterEnabled, } n.groupingInterface.AddEventHandler(appliedToGroupType, n.enqueueAppliedToGroup) n.groupingInterface.AddEventHandler(addressGroupType, n.enqueueAddressGroup) n.groupingInterface.AddEventHandler(internalGroupType, n.enqueueInternalGroup) + n.labelIdentityInterface.AddEventHandler(n.triggerPolicyResyncForLabelIdentityUpdates) // Add handlers for NetworkPolicy events. networkPolicyInformer.Informer().AddEventHandlerWithResyncPeriod( cache.ResourceEventHandlerFuncs{ @@ -1384,6 +1395,15 @@ func (n *NetworkPolicyController) getInternalGroupWorkloads(group *antreatypes.G return pods, ees, nil } +func (n *NetworkPolicyController) triggerPolicyResyncForLabelIdentityUpdates(key string) { + klog.V(2).InfoS("Resyncing policy for LabelIdentity events", "policy", key) + internalNPObj, found, _ := n.internalNetworkPolicyStore.Get(key) + if !found { + return + } + n.enqueueInternalNetworkPolicy(internalNPObj.(*antreatypes.NetworkPolicy).SourceRef) +} + // syncInternalNetworkPolicy retrieves all the AppliedToGroups associated with // itself in order to calculate the Node span for this policy. func (n *NetworkPolicyController) syncInternalNetworkPolicy(key *controlplane.NetworkPolicyReference) error { @@ -1548,7 +1568,9 @@ func (n *NetworkPolicyController) deleteInternalNetworkPolicy(name string) { internalNetworkPolicy := obj.(*antreatypes.NetworkPolicy) n.internalNetworkPolicyStore.Delete(internalNetworkPolicy.Name) n.cleanupOrphanGroups(internalNetworkPolicy) - + if n.multiclusterEnabled && internalNetworkPolicy.SourceRef.Type != controlplane.K8sNetworkPolicy { + n.labelIdentityInterface.DeletePolicySelectors(internalNetworkPolicy.Name) + } // Enqueue AddressGroups previously used by this NetworkPolicy as their span may change due to the removal. for agName := range internalNetworkPolicy.GetAddressGroups() { n.enqueueAddressGroup(agName) diff --git a/pkg/controller/networkpolicy/networkpolicy_controller_test.go b/pkg/controller/networkpolicy/networkpolicy_controller_test.go index 2eb96ec28a3..e31086a789d 100644 --- a/pkg/controller/networkpolicy/networkpolicy_controller_test.go +++ b/pkg/controller/networkpolicy/networkpolicy_controller_test.go @@ -41,6 +41,8 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + fakemcsversioned "antrea.io/antrea/multicluster/pkg/client/clientset/versioned/fake" + mcsinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/apis/controlplane" crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/apis/crd/v1alpha2" @@ -49,6 +51,7 @@ import ( fakeversioned "antrea.io/antrea/pkg/client/clientset/versioned/fake" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/controller/grouping" + "antrea.io/antrea/pkg/controller/labelidentity" "antrea.io/antrea/pkg/controller/networkpolicy/store" antreatypes "antrea.io/antrea/pkg/controller/types" ) @@ -91,14 +94,17 @@ type networkPolicyController struct { informerFactory informers.SharedInformerFactory crdInformerFactory crdinformers.SharedInformerFactory groupingController *grouping.GroupEntityController + labelIdentityController *labelidentity.Controller } // objects is an initial set of K8s objects that is exposed through the client. func newController(objects ...runtime.Object) (*fake.Clientset, *networkPolicyController) { client := newClientset(objects...) crdClient := fakeversioned.NewSimpleClientset() + mcsClient := fakemcsversioned.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, informerDefaultResync) crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync) + mcsInformerFactory := mcsinformers.NewSharedInformerFactory(mcsClient, informerDefaultResync) appliedToGroupStore := store.NewAppliedToGroupStore() addressGroupStore := store.NewAddressGroupStore() internalNetworkPolicyStore := store.NewNetworkPolicyStore() @@ -110,9 +116,14 @@ func newController(objects ...runtime.Object) (*fake.Clientset, *networkPolicyCo informerFactory.Core().V1().Pods(), informerFactory.Core().V1().Namespaces(), crdInformerFactory.Crd().V1alpha2().ExternalEntities()) + labelIndex := labelidentity.NewLabelIdentityIndex() + labelIdentityController := labelidentity.NewLabelIdentityController( + labelIndex, + mcsInformerFactory.Multicluster().V1alpha1().LabelIdentities()) npController := NewNetworkPolicyController(client, crdClient, groupEntityIndex, + labelIndex, informerFactory.Core().V1().Namespaces(), informerFactory.Core().V1().Services(), informerFactory.Networking().V1().NetworkPolicies(), @@ -125,7 +136,8 @@ func newController(objects ...runtime.Object) (*fake.Clientset, *networkPolicyCo addressGroupStore, appliedToGroupStore, internalNetworkPolicyStore, - internalGroupStore) + internalGroupStore, + true) npController.namespaceLister = informerFactory.Core().V1().Namespaces().Lister() npController.namespaceListerSynced = alwaysReady npController.networkPolicyListerSynced = alwaysReady @@ -153,6 +165,7 @@ func newController(objects ...runtime.Object) (*fake.Clientset, *networkPolicyCo informerFactory, crdInformerFactory, groupingController, + labelIdentityController, } } @@ -223,6 +236,7 @@ func newControllerWithoutEventHandler(k8sObjects, crdObjects []runtime.Object) ( informerFactory, crdInformerFactory, nil, + nil, } }