From 52863a5fcee8aa0aee17c8e34e4ad62797fce2d2 Mon Sep 17 00:00:00 2001 From: mhmxs Date: Mon, 8 Jun 2020 19:15:21 +0200 Subject: [PATCH] BGP Peer generation --- Dockerfile | 1 + README.md | 2 +- bgppeer/bgppeer.go | 57 +++++++++++++ config/manager/manager.yaml | 2 + config/rbac/role.yaml | 6 +- .../routereflectorconfig_controller.go | 80 ++++++++++++++++--- datastores/kdd.go | 6 +- go.mod | 1 + main.go | 33 ++++++-- topologies/multi.go | 77 +++++++++++++++++- topologies/single.go | 53 +++++++++++- topologies/topology.go | 21 ++++- 12 files changed, 313 insertions(+), 26 deletions(-) create mode 100644 bgppeer/bgppeer.go diff --git a/Dockerfile b/Dockerfile index 9c8acb7..3e2044c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN go mod download COPY main.go main.go COPY api/ api/ COPY controllers/ controllers/ +COPY bgppeer/ bgppeer/ COPY datastores/ datastores/ COPY topologies/ topologies/ diff --git a/README.md b/README.md index d2d43bf..03928f6 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Build your own image: ## Roadmap - * Etcd data store support (Currently you have to edit [manager](config/manager/manager.yaml)'s yaml manually) + * Etcd data store support * Use custom resource instead of environment variables * Dedicated or preferred node label * Disallow node label diff --git a/bgppeer/bgppeer.go b/bgppeer/bgppeer.go new file mode 100644 index 0000000..7bb7e11 --- /dev/null +++ b/bgppeer/bgppeer.go @@ -0,0 +1,57 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package bgppeer + +import ( + "context" + + calicoApi "github.com/projectcalico/libcalico-go/lib/apis/v3" + calicoClient "github.com/projectcalico/libcalico-go/lib/clientv3" + "github.com/projectcalico/libcalico-go/lib/options" +) + +type BGPPeer struct { + CalicoClient calicoClient.Interface +} + +func (b *BGPPeer) ListBGPPeers() (*calicoApi.BGPPeerList, error) { + return b.CalicoClient.BGPPeers().List(context.Background(), options.ListOptions{}) +} + +func (b *BGPPeer) SaveBGPPeer(peer *calicoApi.BGPPeer) error { + if peer.GetUID() == "" { + if _, err := b.CalicoClient.BGPPeers().Create(context.Background(), peer, options.SetOptions{}); err != nil { + return err + } + } else { + if _, err := b.CalicoClient.BGPPeers().Update(context.Background(), peer, options.SetOptions{}); err != nil { + return err + } + } + + return nil +} + +func (b *BGPPeer) RemoveBGPPeer(peer *calicoApi.BGPPeer) error { + _, err := b.CalicoClient.BGPPeers().Delete(context.Background(), peer.GetName(), options.DeleteOptions{}) + return err +} + +func NewBGPPeer(calicoClient calicoClient.Interface) *BGPPeer { + return &BGPPeer{ + CalicoClient: calicoClient, + } +} diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index db2bbfd..3f059f7 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -32,6 +32,8 @@ spec: value: "incluster" - name: K8S_API_ENDPOINT value: "https://kubernetes.default" + - name: ROUTE_REFLECTOR_TOPOLOGY + value: "multi" image: controller:latest imagePullPolicy: Always name: manager diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f1e6a42..bd0ee52 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -16,10 +16,12 @@ rules: - update - watch - apiGroups: - - projectcalico.org/v3 + - crd.projectcalico.org resources: - - nodes + - bgppeers verbs: + - create + - get - list - update - apiGroups: diff --git a/controllers/routereflectorconfig_controller.go b/controllers/routereflectorconfig_controller.go index 873a201..c83f741 100644 --- a/controllers/routereflectorconfig_controller.go +++ b/controllers/routereflectorconfig_controller.go @@ -20,8 +20,10 @@ import ( "context" "github.com/go-logr/logr" + "github.com/mhmxs/calico-route-reflector-operator/bgppeer" "github.com/mhmxs/calico-route-reflector-operator/datastores" "github.com/mhmxs/calico-route-reflector-operator/topologies" + calicoApi "github.com/projectcalico/libcalico-go/lib/apis/v3" "github.com/prometheus/common/log" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -48,6 +50,10 @@ var ( nodeRevertError = ctrl.Result{} nodeRevertUpdateError = ctrl.Result{} nodeUpdateError = ctrl.Result{} + rrListError = ctrl.Result{} + rrPeerListError = ctrl.Result{} + bgpPeerError = ctrl.Result{} + bgpPeerRemoveError = ctrl.Result{} ) // RouteReflectorConfigReconciler reconciles a RouteReflectorConfig object @@ -56,8 +62,10 @@ type RouteReflectorConfigReconciler struct { CalicoClient calicoClient.Interface Log logr.Logger Scheme *runtime.Scheme + NodeLabelKey string Topology topologies.Topology Datastore datastores.Datastore + BGPPeer bgppeer.BGPPeer } type reconcileImplClient interface { @@ -69,22 +77,22 @@ type reconcileImplClient interface { // +kubebuilder:rbac:groups=route-reflector.calico-route-reflector-operator.mhmxs.github.com,resources=routereflectorconfigs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=route-reflector.calico-route-reflector-operator.mhmxs.github.com,resources=routereflectorconfigs/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;update;watch -// +kubebuilder:rbac:groups="projectcalico.org/v3",resources=nodes,verbs=list;update +// +kubebuilder:rbac:groups="crd.projectcalico.org",resources=bgppeers,verbs=get;list;create;update func (r *RouteReflectorConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { _ = r.Log.WithValues("routereflectorconfig", req.Name) - targetNode := corev1.Node{} - if err := r.Client.Get(context.Background(), req.NamespacedName, &targetNode); err != nil && !errors.IsNotFound(err) { + currentNode := corev1.Node{} + if err := r.Client.Get(context.Background(), req.NamespacedName, ¤tNode); err != nil && !errors.IsNotFound(err) { log.Errorf("Unable to fetch node %s because of %s", req.Name, err.Error()) return nodeGetError, err } else if errors.IsNotFound(err) { log.Debugf("Node not found %s", req.Name) return nodeNotFound, nil - } else if err == nil && r.Topology.IsLabeled(string(targetNode.GetUID()), targetNode.GetLabels()) && targetNode.GetDeletionTimestamp() != nil || - !isNodeReady(&targetNode) || !isNodeSchedulable(&targetNode) { + } else if err == nil && r.Topology.IsRouteReflector(string(currentNode.GetUID()), currentNode.GetLabels()) && currentNode.GetDeletionTimestamp() != nil || + !isNodeReady(¤tNode) || !isNodeSchedulable(¤tNode) { // Node is deleted right now or has some issues, better to remove form RRs - if err := r.removeRRStatus(req, &targetNode); err != nil { + if err := r.removeRRStatus(req, ¤tNode); err != nil { log.Errorf("Unable to cleanup label on %s because of %s", req.Name, err.Error()) return nodeCleanupError, err } @@ -93,7 +101,7 @@ func (r *RouteReflectorConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Resul return nodeCleaned, nil } - listOptions := r.Topology.NewNodeListOptions(targetNode.GetLabels()) + listOptions := r.Topology.NewNodeListOptions(currentNode.GetLabels()) log.Debugf("List options are %v", listOptions) nodeList := corev1.NodeList{} if err := r.Client.List(context.Background(), &nodeList, &listOptions); err != nil { @@ -153,6 +161,50 @@ func (r *RouteReflectorConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Resul log.Infof("Actual number %d is different than expected %d", actualReadyNumber, expectedNumber) } + rrLables := client.HasLabels{r.NodeLabelKey} + rrListOptions := client.ListOptions{} + rrLables.ApplyToList(&rrListOptions) + log.Debugf("RR list options are %v", rrListOptions) + + rrList := corev1.NodeList{} + if err := r.Client.List(context.Background(), &rrList, &rrListOptions); err != nil { + log.Errorf("Unable to list route reflectors because of %s", err.Error()) + return rrListError, err + } + + log.Debugf("Route reflectors are: %v", rrList.Items) + + existingBGPPeers, err := r.BGPPeer.ListBGPPeers() + if err != nil { + log.Errorf("Unable to list BGP peers because of %s", err.Error()) + return rrPeerListError, err + } + + log.Debugf("Existing BGPeers are: %v", existingBGPPeers.Items) + + currentBGPPeers := r.Topology.GenerateBGPPeers(rrList.Items, nodes, existingBGPPeers) + + log.Debugf("Current BGPeers are: %v", currentBGPPeers) + + for _, bp := range currentBGPPeers { + if err := r.BGPPeer.SaveBGPPeer(&bp); err != nil { + log.Errorf("Unable to save BGPPeer because of %s", err.Error()) + return bgpPeerError, err + } + } + + for _, p := range existingBGPPeers.Items { + if !findBGPPeer(p.GetName(), currentBGPPeers) { + log.Debugf("Removing BGPPeer: %s", p.GetName()) + if err := r.BGPPeer.RemoveBGPPeer(&p); err != nil { + log.Errorf("Unable to remove BGPPeer because of %s", err.Error()) + return bgpPeerRemoveError, err + } + } + } + + // TODO check do logs make sense to debug + return finished, nil } @@ -178,7 +230,7 @@ func (r *RouteReflectorConfigReconciler) removeRRStatus(req ctrl.Request, node * } func (r *RouteReflectorConfigReconciler) updateRRStatus(node *corev1.Node, diff int) (bool, error) { - if labeled := r.Topology.IsLabeled(string(node.GetUID()), node.GetLabels()); labeled && diff < 0 { + if labeled := r.Topology.IsRouteReflector(string(node.GetUID()), node.GetLabels()); labeled && diff < 0 { return true, r.Datastore.RemoveRRStatus(node) } else if labeled || diff <= 0 { return false, nil @@ -213,7 +265,7 @@ func (r *RouteReflectorConfigReconciler) collectNodeInfo(allNodes []corev1.Node) filtered[&n] = isReady && isSchedulable if isReady && isSchedulable { readyNodes++ - if r.Topology.IsLabeled(string(n.GetUID()), n.GetLabels()) { + if r.Topology.IsRouteReflector(string(n.GetUID()), n.GetLabels()) { actualReadyNumber++ } } @@ -239,6 +291,16 @@ func isNodeSchedulable(node *corev1.Node) bool { return true } +func findBGPPeer(name string, peers []calicoApi.BGPPeer) bool { + for _, p := range peers { + if p.GetName() == name { + return true + } + } + + return false +} + type eventFilter struct{} func (ef eventFilter) Create(event.CreateEvent) bool { diff --git a/datastores/kdd.go b/datastores/kdd.go index e956e7f..78b7f6e 100644 --- a/datastores/kdd.go +++ b/datastores/kdd.go @@ -21,7 +21,7 @@ import ( ) const ( - routeReflectorClusterIDAnnotation = "projectcalico.org/RouteReflectorClusterID" + RouteReflectorClusterIDAnnotation = "projectcalico.org/RouteReflectorClusterID" ) type KddDataStore struct { @@ -31,7 +31,7 @@ type KddDataStore struct { func (d *KddDataStore) RemoveRRStatus(node *corev1.Node) error { nodeLabelKey, _ := d.topology.GetNodeLabel(string(node.GetUID())) delete(node.Labels, nodeLabelKey) - delete(node.Annotations, routeReflectorClusterIDAnnotation) + delete(node.Annotations, RouteReflectorClusterIDAnnotation) return nil } @@ -41,7 +41,7 @@ func (d *KddDataStore) AddRRStatus(node *corev1.Node) error { node.Labels[labelKey] = labelValue clusterID := d.topology.GetClusterID(string(node.GetUID())) - node.Annotations[routeReflectorClusterIDAnnotation] = clusterID + node.Annotations[RouteReflectorClusterIDAnnotation] = clusterID return nil } diff --git a/go.mod b/go.mod index 1f5e52f..b35c060 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/onsi/gomega v1.8.1 github.com/projectcalico/libcalico-go v1.7.2-0.20200427180741-f197f7370140 github.com/prometheus/common v0.4.1 + google.golang.org/appengine v1.5.0 k8s.io/api v0.17.2 k8s.io/apimachinery v0.17.2 k8s.io/client-go v0.17.2 diff --git a/main.go b/main.go index be6635d..1acde3b 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" routereflectorv1 "github.com/mhmxs/calico-route-reflector-operator/api/v1" + "github.com/mhmxs/calico-route-reflector-operator/bgppeer" "github.com/mhmxs/calico-route-reflector-operator/controllers" "github.com/mhmxs/calico-route-reflector-operator/datastores" "github.com/mhmxs/calico-route-reflector-operator/topologies" @@ -65,6 +66,23 @@ func init() { _ = clientgoscheme.AddToScheme(scheme) _ = routereflectorv1.AddToScheme(scheme) + + // calicoGroupVersion := &schema.GroupVersion{ + // Group: "crd.projectcalico.org", + // Version: "v1", + // } + + // schemeBuilder := runtime.NewSchemeBuilder( + // func(scheme *runtime.Scheme) error { + // scheme.AddKnownTypes( + // *calicoGroupVersion, + // &calicoApi.BGPPeer{}, + // &calicoApi.BGPPeerList{}, + // ) + // return nil + // }) + + // _ = schemeBuilder.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } @@ -109,6 +127,7 @@ func main() { Ration: ratio, } var topology topologies.Topology + // TODO Validation on topology if t, ok := os.LookupEnv("ROUTE_REFLECTOR_TOPOLOGY"); ok && t == "multi" { topology = topologies.NewMultiTopology(topologyConfig) } else { @@ -132,11 +151,15 @@ func main() { } if err = (&controllers.RouteReflectorConfigReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("RouteReflectorConfig"), - Scheme: mgr.GetScheme(), - Topology: topology, - Datastore: datastore, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RouteReflectorConfig"), + Scheme: mgr.GetScheme(), + NodeLabelKey: nodeLabelKey, + Topology: topology, + Datastore: datastore, + BGPPeer: bgppeer.BGPPeer{ + CalicoClient: calicoClient, + }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "RouteReflectorConfig") panic(err) diff --git a/topologies/multi.go b/topologies/multi.go index 9d3efc2..9122ae1 100644 --- a/topologies/multi.go +++ b/topologies/multi.go @@ -17,7 +17,12 @@ package topologies import ( "fmt" + "math/rand" + "strconv" + calicoApi "github.com/projectcalico/libcalico-go/lib/apis/v3" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -28,7 +33,7 @@ type MultiTopology struct { single SingleTopology } -func (t *MultiTopology) IsLabeled(nodeID string, labels map[string]string) bool { +func (t *MultiTopology) IsRouteReflector(nodeID string, labels map[string]string) bool { label, ok := labels[t.NodeLabelKey] return ok && label == t.getNodeLabel(nodeID) } @@ -42,15 +47,72 @@ func (t *MultiTopology) GetNodeLabel(nodeID string) (string, string) { } func (t *MultiTopology) NewNodeListOptions(nodeLabels map[string]string) client.ListOptions { - return t.single.NewNodeListOptions(nodeLabels) + return client.ListOptions{} } func (t *MultiTopology) CalculateExpectedNumber(readyNodes int) int { return t.single.CalculateExpectedNumber(readyNodes) } -func (t *MultiTopology) getNodeLabel(nodeID string) string { - return fmt.Sprintf("%s-%d", t.NodeLabelValue, getRouteReflectorID(nodeID)) +func (t *MultiTopology) GenerateBGPPeers(routeReflectors []corev1.Node, nodes map[*corev1.Node]bool, existingPeers *calicoApi.BGPPeerList) []calicoApi.BGPPeer { + bgpPeerConfigs := []calicoApi.BGPPeer{} + + for n, isReady := range nodes { + if !isReady { + continue + } + + if t.IsRouteReflector(string(n.GetUID()), n.GetLabels()) { + selector := fmt.Sprintf("has(%s)", t.NodeLabelKey) + rrConfig := findBGPPeer(DefaultRouteReflectorMeshName, existingPeers) + if rrConfig == nil { + rrConfig = &calicoApi.BGPPeer{ + TypeMeta: metav1.TypeMeta{ + Kind: calicoApi.KindBGPPeer, + APIVersion: calicoApi.GroupVersionCurrent, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultRouteReflectorMeshName, + }, + } + } + rrConfig.Spec = calicoApi.BGPPeerSpec{ + NodeSelector: "!" + selector, + PeerSelector: selector, + } + + bgpPeerConfigs = append(bgpPeerConfigs, *rrConfig) + + continue + } + + // TODO Do it in a more sophisticaged way + for i := 1; i <= 3; i++ { + rrID := rand.Intn(len(routeReflectors)) + name := fmt.Sprintf(DefaultRouteReflectorClientName, rrID) + rr := getRouteReflectorID(string(routeReflectors[rrID].GetUID())) + + clientConfig := findBGPPeer(DefaultRouteReflectorMeshName, existingPeers) + if clientConfig == nil { + clientConfig = &calicoApi.BGPPeer{ + TypeMeta: metav1.TypeMeta{ + Kind: calicoApi.KindBGPPeer, + APIVersion: calicoApi.GroupVersionCurrent, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + } + clientConfig.Spec = calicoApi.BGPPeerSpec{ + PeerSelector: fmt.Sprintf("%s=='%d'", t.NodeLabelKey, rr), + } + + bgpPeerConfigs = append(bgpPeerConfigs, *clientConfig) + } + } + + return bgpPeerConfigs } func (t *MultiTopology) AddRRSuccess(nodeID string) { @@ -61,6 +123,13 @@ func (t *MultiTopology) RemoveRRSuccess(nodeID string) { delete(routeReflectors, nodeID) } +func (t *MultiTopology) getNodeLabel(nodeID string) string { + if t.NodeLabelValue == "" { + return strconv.Itoa(getRouteReflectorID(nodeID)) + } + return fmt.Sprintf("%s-%d", t.NodeLabelValue, getRouteReflectorID(nodeID)) +} + // TODO this method has several performance issues, needs to fix later func getRouteReflectorID(nodeID string) int { if existing, ok := routeReflectors[nodeID]; ok { diff --git a/topologies/single.go b/topologies/single.go index 5357b40..36cfd83 100644 --- a/topologies/single.go +++ b/topologies/single.go @@ -19,6 +19,9 @@ import ( "fmt" "math" + calicoApi "github.com/projectcalico/libcalico-go/lib/apis/v3" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,7 +31,7 @@ type SingleTopology struct { Config } -func (t *SingleTopology) IsLabeled(_ string, labels map[string]string) bool { +func (t *SingleTopology) IsRouteReflector(_ string, labels map[string]string) bool { label, ok := labels[t.NodeLabelKey] return ok && label == t.NodeLabelValue } @@ -70,6 +73,54 @@ func (t *SingleTopology) CalculateExpectedNumber(readyNodes int) int { return int(exp) } +func (t *SingleTopology) GenerateBGPPeers(_ []corev1.Node, _ map[*corev1.Node]bool, existingPeers *calicoApi.BGPPeerList) []calicoApi.BGPPeer { + bgpPeerConfigs := []calicoApi.BGPPeer{} + + selector := fmt.Sprintf("has(%s)", t.NodeLabelKey) + + rrConfig := findBGPPeer(DefaultRouteReflectorMeshName, existingPeers) + if rrConfig == nil { + rrConfig = &calicoApi.BGPPeer{ + TypeMeta: metav1.TypeMeta{ + Kind: calicoApi.KindBGPPeer, + APIVersion: calicoApi.GroupVersionCurrent, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultRouteReflectorMeshName, + }, + } + } + rrConfig.Spec = calicoApi.BGPPeerSpec{ + NodeSelector: "!" + selector, + PeerSelector: selector, + } + + bgpPeerConfigs = append(bgpPeerConfigs, *rrConfig) + + clientConfigName := fmt.Sprintf(DefaultRouteReflectorClientName, 1) + + clientConfig := findBGPPeer(clientConfigName, existingPeers) + if clientConfig == nil { + clientConfig = &calicoApi.BGPPeer{ + TypeMeta: metav1.TypeMeta{ + Kind: calicoApi.KindBGPPeer, + APIVersion: calicoApi.GroupVersionCurrent, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clientConfigName, + }, + } + } + clientConfig.Spec = calicoApi.BGPPeerSpec{ + NodeSelector: selector, + PeerSelector: selector, + } + + bgpPeerConfigs = append(bgpPeerConfigs, *clientConfig) + + return bgpPeerConfigs +} + func (t *SingleTopology) AddRRSuccess(string) { } diff --git a/topologies/topology.go b/topologies/topology.go index cc1af3c..fd25fed 100644 --- a/topologies/topology.go +++ b/topologies/topology.go @@ -17,15 +17,24 @@ limitations under the License. package topologies import ( + calicoApi "github.com/projectcalico/libcalico-go/lib/apis/v3" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + // TODO Make it configurable + DefaultRouteReflectorMeshName = "rrs-to-rrs" + DefaultRouteReflectorClientName = "peer-to-rrs-%d" +) + type Topology interface { - IsLabeled(string, map[string]string) bool + IsRouteReflector(string, map[string]string) bool GetClusterID(string) string GetNodeLabel(string) (string, string) NewNodeListOptions(labels map[string]string) client.ListOptions CalculateExpectedNumber(int) int + GenerateBGPPeers([]corev1.Node, map[*corev1.Node]bool, *calicoApi.BGPPeerList) []calicoApi.BGPPeer AddRRSuccess(string) RemoveRRSuccess(string) } @@ -39,3 +48,13 @@ type Config struct { Max int Ration float64 } + +func findBGPPeer(name string, peers *calicoApi.BGPPeerList) *calicoApi.BGPPeer { + for _, p := range peers.Items { + if p.GetName() == name { + return &p + } + } + + return nil +}