From 5532d458f43c11a7a26e074ec0c2e8da9e84f6ec Mon Sep 17 00:00:00 2001 From: mhmxs Date: Fri, 8 May 2020 16:17:25 +0200 Subject: [PATCH] Version alpha v0.0.1 --- Makefile | 38 +++- PROJECT | 4 + README.md | 26 ++- api/v1/groupversion_info.go | 36 ++++ api/v1/routereflectorconfig_types.go | 63 ++++++ api/v1/zz_generated.deepcopy.go | 114 +++++++++++ ...hmxs.github.com_routereflectorconfigs.yaml | 57 ++++++ config/crd/kustomization.yaml | 21 ++ config/crd/kustomizeconfig.yaml | 17 ++ .../cainjection_in_routereflectorconfigs.yaml | 8 + .../webhook_in_routereflectorconfigs.yaml | 17 ++ config/manager/kustomization.yaml | 6 + config/manager/manager.yaml | 2 + config/rbac/kustomization.yaml | 18 +- config/rbac/role.yaml | 37 ++++ .../routereflectorconfig_editor_role.yaml | 24 +++ ...tereflectorconfig_editor_role_binding.yaml | 12 ++ .../routereflectorconfig_viewer_role.yaml | 20 ++ ...tereflectorconfig_viewer_role_binding.yaml | 12 ++ ...ute-reflector_v1_routereflectorconfig.yaml | 7 + .../routereflectorconfig_controller.go | 184 ++++++++++++++++++ controllers/suite_test.go | 81 ++++++++ go.mod | 5 + go.sum | 4 + k | 0 main.go | 88 ++++++++- 26 files changed, 883 insertions(+), 18 deletions(-) create mode 100644 api/v1/groupversion_info.go create mode 100644 api/v1/routereflectorconfig_types.go create mode 100644 api/v1/zz_generated.deepcopy.go create mode 100644 config/crd/bases/route-reflector.calico-route-reflector-operator.mhmxs.github.com_routereflectorconfigs.yaml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/crd/kustomizeconfig.yaml create mode 100644 config/crd/patches/cainjection_in_routereflectorconfigs.yaml create mode 100644 config/crd/patches/webhook_in_routereflectorconfigs.yaml create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/routereflectorconfig_editor_role.yaml create mode 100644 config/rbac/routereflectorconfig_editor_role_binding.yaml create mode 100644 config/rbac/routereflectorconfig_viewer_role.yaml create mode 100644 config/rbac/routereflectorconfig_viewer_role_binding.yaml create mode 100644 config/samples/route-reflector_v1_routereflectorconfig.yaml create mode 100644 controllers/routereflectorconfig_controller.go create mode 100644 controllers/suite_test.go create mode 100644 k diff --git a/Makefile b/Makefile index 5da22b5..981e88e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ +GIT_COMMIT_SHA:=$(shell git rev-parse HEAD 2>/dev/null) + +IMG_REPO ?= quay.io/mhmxs +IMG_NAME ?= calico-route-reflector-controller +IMG_VERSION ?= latest -# Image URL to use all building/pushing image targets -IMG ?= controller:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" @@ -13,6 +16,9 @@ endif all: manager +_calculate-build-number: + $(eval export CONTAINER_VERSION?=$(GIT_COMMIT_SHA)-$(shell date "+%s")) + # Run tests test: generate fmt vet manifests go test ./... -coverprofile cover.out @@ -35,9 +41,19 @@ uninstall: manifests # Deploy controller in the configured Kubernetes cluster in ~/.kube/config deploy: manifests - cd config/manager && kustomize edit set image controller=${IMG} + cd config/manager && kustomize edit set image controller=$(IMG_REPO)/$(IMG_NAME):$(IMG_VERSION) kustomize build config/default | kubectl apply -f - +# Undeploy deletes resources +undeploy: + kustomize build config/default | kubectl delete -f - + +logs: + kubectl logs -n calico-route-reflector-operator-system $$(kubectl get po -A | grep calico-route-reflector-operator-system | awk '{print $$2}') manager + +logs-f: + kubectl logs -f -n calico-route-reflector-operator-system $$(kubectl get po -A | grep calico-route-reflector-operator-system | awk '{print $$2}') manager + # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases @@ -55,12 +71,20 @@ generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." # Build the docker image -docker-build: test - docker build . -t ${IMG} +docker-build: _calculate-build-number test + docker build -t $(IMG_NAME):$(IMG_VERSION) . + +dev-docker-build: _calculate-build-number test + docker build -t $(IMG_NAME):$(CONTAINER_VERSION) . # Push the docker image -docker-push: - docker push ${IMG} +docker-push: docker-build + docker tag $(IMG_NAME):$(IMG_VERSION) $(IMG_REPO)/$(IMG_NAME):$(IMG_VERSION) + docker push $(IMG_REPO)/$(IMG_NAME):$(IMG_VERSION) + +dev-docker-push: docker-build + docker tag $(IMG_NAME):$(CONTAINER_VERSION) $(IMG_REPO)/$(IMG_NAME):$(CONTAINER_VERSION) + docker push $(IMG_REPO)/$(IMG_NAME):$(CONTAINER_VERSION) # find or download controller-gen # download controller-gen if necessary diff --git a/PROJECT b/PROJECT index 65bb0a8..67439a7 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ domain: calico-route-reflector-operator.mhmxs.github.com repo: github.com/mhmxs/calico-route-reflector-operator +resources: +- group: route-reflector + kind: RouteReflectorConfig + version: v1 version: "2" diff --git a/README.md b/README.md index 0f84935..8a72111 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# calico-route-reflector-operator -This Kubernetes operator can monitor and scale Calico route refloctor pods based on cluster size. +[![Go Report Card](https://goreportcard.com/badge/github.com/mhmxs/calico-route-reflector-operator)](https://goreportcard.com/report/mhmxs/calico-route-reflector-operator) [![Active](http://img.shields.io/badge/Status-Active-green.svg)](https://github.com/mhmxs/calico-route-reflector-operator) [![PR's Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat)](https://github.com/mhmxs/calico-route-reflector-operator/pulls) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +# Calico Route Reflector Operator + +This project is work in progress !!! +Use your own risk !!! + +This Kubernetes operator can monitor and scale Calico route refloctor pods based on cluster size. The operator has a few environment variable: + * `ROUTE_REFLECTOR_MIN` Minimum number of route reflector pods, default `3` + * `ROUTE_REFLECTOR_MAX` Maximum number of route reflector pods, default `10` + * `ROUTE_REFLECTOR_RATIO` Node / route reflector pod ratio, default `0.2` (`100 * 0.2 = 20`) + * `ROUTE_REFLECTOR_NODE_LABEL` Node label of the route reflector nodes, default `calico-route-reflector=` +// * `ROUTE_REFLECTOR_ZONE_LABEL` Node label of the zone, default `""` +// * `ROUTE_REFLECTOR_PROTECTED` Node label of the nodes where route reflector pods are not allowed, default `""` + +During the `api/core/v1/Node` reconcile phases it calculates the right number of route refloctor pods by multiply the number of nodes with the given ratio. +It updates the route reflector replicas to the expected number. +// If the given `ROUTE_REFLECTOR_ZONE_LABEL` is not empty, the operator balancing route reflector pods between the zones. + +## Build + +This is a standard Kubebuilder opertor so building and deploying process is similar as a (stock Kubebuilder project)[https://book.kubebuilder.io/cronjob-tutorial/running.html]. + +`IMG_REPO=[IMG_REPO] IMG_NAME=[IMG_NAME] IMG_VERSION=[IMG_VERSION] make docker-push install deploy` \ No newline at end of file diff --git a/api/v1/groupversion_info.go b/api/v1/groupversion_info.go new file mode 100644 index 0000000..6b9d31f --- /dev/null +++ b/api/v1/groupversion_info.go @@ -0,0 +1,36 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1 contains API Schema definitions for the route-reflector v1 API group +// +kubebuilder:object:generate=true +// +groupName=route-reflector.calico-route-reflector-operator.mhmxs.github.com +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "route-reflector.calico-route-reflector-operator.mhmxs.github.com", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1/routereflectorconfig_types.go b/api/v1/routereflectorconfig_types.go new file mode 100644 index 0000000..bc9a11f --- /dev/null +++ b/api/v1/routereflectorconfig_types.go @@ -0,0 +1,63 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RouteReflectorConfigSpec defines the desired state of RouteReflectorConfig +type RouteReflectorConfigSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of RouteReflectorConfig. Edit RouteReflectorConfig_types.go to remove/update + Foo string `json:"foo,omitempty"` +} + +// RouteReflectorConfigStatus defines the observed state of RouteReflectorConfig +type RouteReflectorConfigStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true + +// RouteReflectorConfig is the Schema for the routereflectorconfigs API +type RouteReflectorConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RouteReflectorConfigSpec `json:"spec,omitempty"` + Status RouteReflectorConfigStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// RouteReflectorConfigList contains a list of RouteReflectorConfig +type RouteReflectorConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RouteReflectorConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RouteReflectorConfig{}, &RouteReflectorConfigList{}) +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go new file mode 100644 index 0000000..24da48c --- /dev/null +++ b/api/v1/zz_generated.deepcopy.go @@ -0,0 +1,114 @@ +// +build !ignore_autogenerated + +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouteReflectorConfig) DeepCopyInto(out *RouteReflectorConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteReflectorConfig. +func (in *RouteReflectorConfig) DeepCopy() *RouteReflectorConfig { + if in == nil { + return nil + } + out := new(RouteReflectorConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RouteReflectorConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouteReflectorConfigList) DeepCopyInto(out *RouteReflectorConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RouteReflectorConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteReflectorConfigList. +func (in *RouteReflectorConfigList) DeepCopy() *RouteReflectorConfigList { + if in == nil { + return nil + } + out := new(RouteReflectorConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RouteReflectorConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouteReflectorConfigSpec) DeepCopyInto(out *RouteReflectorConfigSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteReflectorConfigSpec. +func (in *RouteReflectorConfigSpec) DeepCopy() *RouteReflectorConfigSpec { + if in == nil { + return nil + } + out := new(RouteReflectorConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouteReflectorConfigStatus) DeepCopyInto(out *RouteReflectorConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteReflectorConfigStatus. +func (in *RouteReflectorConfigStatus) DeepCopy() *RouteReflectorConfigStatus { + if in == nil { + return nil + } + out := new(RouteReflectorConfigStatus) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/route-reflector.calico-route-reflector-operator.mhmxs.github.com_routereflectorconfigs.yaml b/config/crd/bases/route-reflector.calico-route-reflector-operator.mhmxs.github.com_routereflectorconfigs.yaml new file mode 100644 index 0000000..3a56e86 --- /dev/null +++ b/config/crd/bases/route-reflector.calico-route-reflector-operator.mhmxs.github.com_routereflectorconfigs.yaml @@ -0,0 +1,57 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: routereflectorconfigs.route-reflector.calico-route-reflector-operator.mhmxs.github.com +spec: + group: route-reflector.calico-route-reflector-operator.mhmxs.github.com + names: + kind: RouteReflectorConfig + listKind: RouteReflectorConfigList + plural: routereflectorconfigs + singular: routereflectorconfig + scope: Namespaced + validation: + openAPIV3Schema: + description: RouteReflectorConfig is the Schema for the routereflectorconfigs + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RouteReflectorConfigSpec defines the desired state of RouteReflectorConfig + properties: + foo: + description: Foo is an example field of RouteReflectorConfig. Edit RouteReflectorConfig_types.go + to remove/update + type: string + type: object + status: + description: RouteReflectorConfigStatus defines the observed state of RouteReflectorConfig + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..d29a064 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,21 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/route-reflector.calico-route-reflector-operator.mhmxs.github.com_routereflectorconfigs.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_routereflectorconfigs.yaml +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_routereflectorconfigs.yaml +# +kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..6f83d9a --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,17 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_routereflectorconfigs.yaml b/config/crd/patches/cainjection_in_routereflectorconfigs.yaml new file mode 100644 index 0000000..f1959ed --- /dev/null +++ b/config/crd/patches/cainjection_in_routereflectorconfigs.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: routereflectorconfigs.route-reflector.calico-route-reflector-operator.mhmxs.github.com diff --git a/config/crd/patches/webhook_in_routereflectorconfigs.yaml b/config/crd/patches/webhook_in_routereflectorconfigs.yaml new file mode 100644 index 0000000..5949e79 --- /dev/null +++ b/config/crd/patches/webhook_in_routereflectorconfigs.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: routereflectorconfigs.route-reflector.calico-route-reflector-operator.mhmxs.github.com +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b8..8265c1c 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/mhmxs/calico-route-reflector-controller + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b6c85a5..1f021dd 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -28,6 +28,7 @@ spec: args: - --enable-leader-election image: controller:latest + imagePullPolicy: Always name: manager resources: limits: @@ -36,4 +37,5 @@ spec: requests: cpu: 100m memory: 20Mi + serviceAccountName: default terminationGracePeriodSeconds: 10 diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 66c2833..0d86ace 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -3,10 +3,14 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +# # Comment the following 4 lines if you want to disable +# # the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# # which protects your /metrics endpoint. +# - auth_proxy_service.yaml +# - auth_proxy_role.yaml +# - auth_proxy_role_binding.yaml +# - auth_proxy_client_clusterrole.yaml +# - routereflectorconfig_editor_role.yaml +# - routereflectorconfig_editor_role_binding.yaml +# - routereflectorconfig_viewer_role.yaml +# - routereflectorconfig_viewer_role_binding.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..28b6e6a --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,37 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - update + - watch +- apiGroups: + - route-reflector.calico-route-reflector-operator.mhmxs.github.com + resources: + - routereflectorconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route-reflector.calico-route-reflector-operator.mhmxs.github.com + resources: + - routereflectorconfigs/status + verbs: + - get + - patch + - update diff --git a/config/rbac/routereflectorconfig_editor_role.yaml b/config/rbac/routereflectorconfig_editor_role.yaml new file mode 100644 index 0000000..44d718c --- /dev/null +++ b/config/rbac/routereflectorconfig_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit routereflectorconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: routereflectorconfig-editor-role +rules: +- apiGroups: + - route-reflector.calico-route-reflector-operator.mhmxs.github.com + resources: + - routereflectorconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - route-reflector.calico-route-reflector-operator.mhmxs.github.com + resources: + - routereflectorconfigs/status + verbs: + - get diff --git a/config/rbac/routereflectorconfig_editor_role_binding.yaml b/config/rbac/routereflectorconfig_editor_role_binding.yaml new file mode 100644 index 0000000..dbce41d --- /dev/null +++ b/config/rbac/routereflectorconfig_editor_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: routereflectorconfig-editor-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: routereflectorconfig-editor-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/routereflectorconfig_viewer_role.yaml b/config/rbac/routereflectorconfig_viewer_role.yaml new file mode 100644 index 0000000..343fbbd --- /dev/null +++ b/config/rbac/routereflectorconfig_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view routereflectorconfigs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: routereflectorconfig-viewer-role +rules: +- apiGroups: + - route-reflector.calico-route-reflector-operator.mhmxs.github.com + resources: + - routereflectorconfigs + verbs: + - get + - list + - watch +- apiGroups: + - route-reflector.calico-route-reflector-operator.mhmxs.github.com + resources: + - routereflectorconfigs/status + verbs: + - get diff --git a/config/rbac/routereflectorconfig_viewer_role_binding.yaml b/config/rbac/routereflectorconfig_viewer_role_binding.yaml new file mode 100644 index 0000000..46f36f9 --- /dev/null +++ b/config/rbac/routereflectorconfig_viewer_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: routereflectorconfig-viewer-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: routereflectorconfig-viewer-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/samples/route-reflector_v1_routereflectorconfig.yaml b/config/samples/route-reflector_v1_routereflectorconfig.yaml new file mode 100644 index 0000000..5ad1c80 --- /dev/null +++ b/config/samples/route-reflector_v1_routereflectorconfig.yaml @@ -0,0 +1,7 @@ +apiVersion: route-reflector.calico-route-reflector-operator.mhmxs.github.com/v1 +kind: RouteReflectorConfig +metadata: + name: routereflectorconfig-sample +spec: + # Add fields here + foo: bar diff --git a/controllers/routereflectorconfig_controller.go b/controllers/routereflectorconfig_controller.go new file mode 100644 index 0000000..3aa507b --- /dev/null +++ b/controllers/routereflectorconfig_controller.go @@ -0,0 +1,184 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "math" + "strings" + + "github.com/go-logr/logr" + "github.com/prometheus/common/log" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +var ( + nodeUnderDelete = ctrl.Result{} + finished = ctrl.Result{} + + nodeGetError = ctrl.Result{} + nodeListError = ctrl.Result{} + nodeUpdateError = ctrl.Result{} +) + +type RouteReflectorConfig struct { + Min int + Max int + Ration float64 + NodeLabel string +} + +// RouteReflectorConfigReconciler reconciles a RouteReflectorConfig object +type RouteReflectorConfigReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + config RouteReflectorConfig +} + +type reconcileImplClient interface { + Get(context.Context, client.ObjectKey, runtime.Object) error + Update(context.Context, runtime.Object, ...client.UpdateOption) error + List(context.Context, runtime.Object, ...client.ListOption) error +} + +type reconcileImplParams struct { + request ctrl.Request + client reconcileImplClient + config RouteReflectorConfig +} + +// +kubebuilder:rbac:groups=route-reflector.calico-route-reflector-operator.mhmxs.github.com,resources=routereflectorconfigs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route-reflector.calico-route-reflector-operator.mhmxs.github.com,resources=routereflectorconfigs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;update;watch + +func (r *RouteReflectorConfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + _ = r.Log.WithValues("routereflectorconfig", req.NamespacedName) + + return reconcileImpl(&reconcileImplParams{ + request: req, + client: r.Client, + config: r.config, + }) +} + +func reconcileImpl(params *reconcileImplParams) (ctrl.Result, error) { + node := &corev1.Node{} + err := params.client.Get(context.Background(), params.request.NamespacedName, node) + + if err != nil && !errors.IsNotFound(err) { + log.Errorf("Unable to fetch node %s reason %s", params.request.NamespacedName, err.Error()) + return nodeGetError, err + } else if err == nil && node.GetDeletionTimestamp() != nil { + return nodeUnderDelete, nil + } + + nodes := &corev1.NodeList{} + if err := params.client.List(context.Background(), nodes, &client.ListOptions{}); err != nil { + log.Errorf("Unable to list nodes ,reason %s", err.Error()) + return nodeListError, err + } + + expectedNumber := int(math.Round(float64(len(nodes.Items)) * params.config.Ration)) + if expectedNumber < params.config.Min { + expectedNumber = params.config.Min + } else if expectedNumber > params.config.Max { + expectedNumber = params.config.Max + } + log.Infof("Expected number of route reflector pods are %d", expectedNumber) + + key, value := getKeyValue(params.config.NodeLabel) + actualNumber := 0 + for _, n := range nodes.Items { + if isLabeled(n.GetLabels(), key, value) { + actualNumber++ + } + } + log.Infof("Actual number of route reflector pods are %d", actualNumber) + + if expectedNumber != actualNumber { + for _, n := range nodes.Items { + if expectedNumber == actualNumber { + break + } + labeled := isLabeled(n.GetLabels(), key, value) + if expectedNumber > actualNumber && !labeled { + log.Infof("Label node %s as route reflector", n.GetName()) + n.Labels[key] = value + actualNumber++ + } else if expectedNumber < actualNumber && labeled { + log.Infof("Remove node %s role route reflector", n.GetName()) + delete(n.Labels, key) + actualNumber-- + } else { + continue + } + + if err = params.client.Update(context.Background(), &n); err != nil { + log.Errorf("Unable to update node %s, reason %s", params.request.NamespacedName, err.Error()) + return nodeUpdateError, err + } + } + } + + return finished, nil +} + +func getKeyValue(label string) (string, string) { + keyValue := strings.Split(label, "=") + if len(keyValue) == 1 { + keyValue[1] = "" + } + + return keyValue[0], keyValue[1] +} + +func isLabeled(labels map[string]string, key, value string) bool { + label, ok := labels[key] + return ok && label == value +} + +type eventFilter struct{} + +func (ef eventFilter) Create(event.CreateEvent) bool { + return true +} + +func (ef eventFilter) Delete(e event.DeleteEvent) bool { + return true +} + +func (ef eventFilter) Update(event.UpdateEvent) bool { + return false +} + +func (ef eventFilter) Generic(event.GenericEvent) bool { + return false +} + +func (r *RouteReflectorConfigReconciler) SetupWithManager(mgr ctrl.Manager, config RouteReflectorConfig) error { + r.config = config + return ctrl.NewControllerManagedBy(mgr). + WithEventFilter(eventFilter{}). + For(&corev1.Node{}). + Complete(r) +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go new file mode 100644 index 0000000..a2d542a --- /dev/null +++ b/controllers/suite_test.go @@ -0,0 +1,81 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + routereflectorv1 "github.com/mhmxs/calico-route-reflector-operator/api/v1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Controller Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func(done Done) { + logf.SetLogger(zap.LoggerTo(GinkgoWriter, true)) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(cfg).ToNot(BeNil()) + + err = routereflectorv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(k8sClient).ToNot(BeNil()) + + close(done) +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).ToNot(HaveOccurred()) +}) diff --git a/go.mod b/go.mod index 5d1e9aa..21b72f4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,11 @@ module github.com/mhmxs/calico-route-reflector-operator go 1.13 require ( + github.com/go-logr/logr v0.1.0 + github.com/onsi/ginkgo v1.11.0 + github.com/onsi/gomega v1.8.1 + github.com/prometheus/common v0.4.1 + k8s.io/api v0.17.2 k8s.io/apimachinery v0.17.2 k8s.io/client-go v0.17.2 sigs.k8s.io/controller-runtime v0.5.0 diff --git a/go.sum b/go.sum index 23fff21..3d27832 100644 --- a/go.sum +++ b/go.sum @@ -19,7 +19,9 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -247,6 +249,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -391,6 +394,7 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/k b/k new file mode 100644 index 0000000..e69de29 diff --git a/main.go b/main.go index d8d19ac..7cba5e2 100644 --- a/main.go +++ b/main.go @@ -17,17 +17,32 @@ limitations under the License. package main import ( + "errors" "flag" + "fmt" "os" + "strconv" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + routereflectorv1 "github.com/mhmxs/calico-route-reflector-operator/api/v1" + "github.com/mhmxs/calico-route-reflector-operator/controllers" + "github.com/prometheus/common/log" // +kubebuilder:scaffold:imports ) +const ( + routeReflectorMin = 3 + routeReflectorMax = 10 + routeReflectorRatio = 0.2 + routeReflectorMaxRatio = 0.5 + routeReflectorLabel = "calico-route-reflector=" +) + var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") @@ -36,10 +51,18 @@ var ( func init() { _ = clientgoscheme.AddToScheme(scheme) + _ = routereflectorv1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } func main() { + defer func() { + if r := recover(); r != nil { + log.Error(fmt.Errorf("%v", r), "") + os.Exit(1) + } + }() + var metricsAddr string var enableLeaderElection bool flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -59,14 +82,75 @@ func main() { }) if err != nil { setupLog.Error(err, "unable to start manager") - os.Exit(1) + panic(err) } + min, max, ratio, nodeLabel := parseEnv() + + if err = (&controllers.RouteReflectorConfigReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RouteReflectorConfig"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr, controllers.RouteReflectorConfig{ + Min: min, + Max: max, + Ration: ratio, + NodeLabel: nodeLabel, + }); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RouteReflectorConfig") + panic(err) + } // +kubebuilder:scaffold:builder setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") - os.Exit(1) + panic(err) } } + +func parseEnv() (int, int, float64, string) { + var err error + min := routeReflectorMin + if v, ok := os.LookupEnv("ROUTE_REFLECTOR_MIN"); ok { + min, err = strconv.Atoi(v) + if err != nil { + setupLog.Error(err, "ROUTE_REFLECTOR_MIN is not an integer") + panic(err) + } else if min < 3 || min > 999 { + err = errors.New("ROUTE_REFLECTOR_MIN must be positive number between 3 and 999") + setupLog.Error(err, err.Error()) + panic(err) + } + } + max := routeReflectorMax + if v, ok := os.LookupEnv("ROUTE_REFLECTOR_MAX"); ok { + max, err = strconv.Atoi(v) + if err != nil { + setupLog.Error(err, "ROUTE_REFLECTOR_MAX is not an integer") + panic(err) + } else if max < 5 || max > 2500 { + err = errors.New("ROUTE_REFLECTOR_MIN must be positive number between 5 and 2500") + setupLog.Error(err, err.Error()) + panic(err) + } + } + ratio := routeReflectorRatio + if v, ok := os.LookupEnv("ROUTE_REFLECTOR_RATIO"); ok { + ratio, err = strconv.ParseFloat(v, 32) + if err != nil { + setupLog.Error(err, "ROUTE_REFLECTOR_RATIO is not a float") + panic(err) + } else if ratio > routeReflectorMaxRatio { + err = fmt.Errorf("ROUTE_REFLECTOR_RATIO is bigger than %f", routeReflectorMaxRatio) + setupLog.Error(err, err.Error()) + panic(err) + } + } + nodeLabel := routeReflectorLabel + if v, ok := os.LookupEnv("ROUTE_REFLECTOR_NODE_LABEL"); ok { + nodeLabel = v + } + + return min, max, ratio, nodeLabel +}