Add Egress-Controller

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.22 AS builder
+FROM ghcr.io/cybozu/golang:1.22-jammy AS builder
 ARG TARGETOS
 ARG TARGETARCH
 
@@ -12,22 +12,15 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY cmd/main.go cmd/main.go
+COPY cmd/egress-controller/main.go cmd/egress-controller/main.go
 COPY api/ api/
 COPY internal/controller/ internal/controller/
 
-# Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o egress-controller cmd/egress-controller/main.go
 
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
+FROM ghcr.io/cybozu/ubuntu:22.04
 WORKDIR /
-COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/egress-controller .
 USER 65532:65532
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/egress-controller"]

Makefile

@@ -44,8 +44,9 @@ help: ## Display this help.
 ##@ Development
 
 .PHONY: manifests
-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) rbac:roleName=egress-controller-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(YQ) -i 'del(.spec.versions.[].schema.openAPIV3Schema.properties.spec.properties.template | .. |select(key == "description"))' config/crd/bases/pona.cybozu.com_egresses.yaml
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -59,10 +60,18 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
+.PHONY: mod
+mod: ## Run go mod tidy against code.
+	go mod tidy
+
 .PHONY: test
-test: manifests generate fmt vet envtest ## Run tests.
+test: vet envtest check-generate ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
+.PHONY: check-generate
+check-generate: manifests generate fmt mod 
+	git diff --exit-code --name-only
+
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
@@ -79,11 +88,11 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 ##@ Build
 
 .PHONY: build
-build: manifests generate fmt vet ## Build manager binary.
+build: manifests generate fmt vet mod ## Build manager binary.
 	go build -o bin/egress-controller cmd/egress-controller/main.go
 
 .PHONY: run
-run: manifests generate fmt vet ## Run a controller from your host.
+run: manifests generate fmt vet mod ## Run a controller from your host.
 	go run ./cmd/egress-controller/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
@@ -156,12 +165,16 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
+YQ = $(LOCALBIN)/yq
+WGET_OPTIONS := --retry-on-http-error=503 --retry-connrefused --no-verbose
+WGET = wget $(WGET_OPTIONS)
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.4.2
 CONTROLLER_TOOLS_VERSION ?= v0.15.0
 ENVTEST_VERSION ?= release-0.18
 GOLANGCI_LINT_VERSION ?= v1.59.1
+YQ_VERSION ?= 4.44.3
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -183,6 +196,15 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
 
+
+.PHONY: yq
+yq: $(YQ)
+$(YQ): $(LOCALBIN)
+	$(WGET) -O yq.tar.gz https://github.com/mikefarah/yq/releases/download/v$(YQ_VERSION)/yq_linux_amd64.tar.gz
+	tar -C $(LOCALBIN)/ -zxf yq.tar.gz ./yq_linux_amd64 -O > $@
+	rm -f yq.tar.gz
+	chmod +x $@
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed

PROJECT

@@ -2,9 +2,19 @@
 # This file is used to track the info used to scaffold your project
 # and allow the plugins properly work.
 # More info: https://book.kubebuilder.io/reference/project-config.html
-domain: pona.cybozu.com
+domain: cybozu.com
 layout:
 - go.kubebuilder.io/v4
-projectName: tmp
+projectName: pona
 repo: github.com/cybozu-go/pona
+resources:
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: cybozu.com
+  group: pona
+  kind: Egress
+  path: github.com/cybozu-go/pona/api/v1beta1
+  version: v1beta1
 version: "3"

api/v1beta1/egress_types.go

@@ -0,0 +1,132 @@
+package v1beta1
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// EgressSpec defines the desired state of Egress
+type EgressSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Destinations is a list of IP networks in CIDR format.
+	// +kubebuilder:validation:MinItems=1
+	Destinations []string `json:"destinations"`
+
+	// Replicas is the desired number of egress (SNAT) pods.
+	// Defaults to 1.
+	// +kubebuilder:default=1
+	// +kubebuilder:validation:Minimum=1
+	// +optional
+	Replicas int32 `json:"replicas"`
+
+	// Strategy describes how to replace existing pods with new ones.
+	// Ref. https://pkg.go.dev/k8s.io/api/apps/v1?tab=doc#DeploymentStrategy
+	// +optional
+	Strategy *appsv1.DeploymentStrategy `json:"strategy,omitempty"`
+
+	// Template is an optional template for egress pods.
+	// A container named "egress" is special.  It is the main container of
+	// egress pods and usually is not meant to be modified.
+	// +optional
+	Template *EgressPodTemplate `json:"template,omitempty"`
+
+	// SessionAffinity is to specify the same field of Service for the Egress.
+	// However, the default is changed from None to ClientIP.
+	// Ref. https://pkg.go.dev/k8s.io/api/core/v1?tab=doc#ServiceSpec
+	// +kubebuilder:validation:Enum=ClientIP;None
+	// +kubebuilder:default=None
+	// +optional
+	SessionAffinity corev1.ServiceAffinity `json:"sessionAffinity,omitempty"`
+
+	// SessionAffinityConfig is to specify the same field of Service for Egress.
+	// Ref. https://pkg.go.dev/k8s.io/api/core/v1?tab=doc#ServiceSpec
+	// +optional
+	SessionAffinityConfig *corev1.SessionAffinityConfig `json:"sessionAffinityConfig,omitempty"`
+
+	// PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT pods.
+	// +optional
+	PodDisruptionBudget *EgressPDBSpec `json:"podDisruptionBudget,omitempty"`
+}
+
+// EgressPodTemplate defines pod template for Egress
+//
+// This is almost the same as corev1.PodTemplate but is simplified to
+// workaround JSON patch issues.
+type EgressPodTemplate struct {
+	// Metadata defines optional labels and annotations
+	// +optional
+	Metadata `json:"metadata,omitempty"`
+
+	// Spec defines the pod template spec.
+	// +optional
+	Spec corev1.PodSpec `json:"spec,omitempty"`
+}
+
+// EgressPDB defines PDB for Egress
+type EgressPDBSpec struct {
+	// MinAvailable is the minimum number of pods that must be available at any given time.
+	// +optional
+	MinAvailable *intstr.IntOrString `json:"minAvailable,omitempty"`
+
+	// MaxUnavailable is the maximum number of pods that can be unavailable at any given time.
+	// +optional
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+}
+
+// Metadata defines a simplified version of ObjectMeta.
+type Metadata struct {
+	// Annotations are optional annotations
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+
+	// Labels are optional labels
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+}
+
+// EgressStatus defines the observed state of Egress
+type EgressStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Replicas is copied from the underlying Deployment's status.replicas.
+	// +optional
+	Replicas int32 `json:"replicas,omitempty"`
+
+	// Selector is a serialized label selector in string form.
+	Selector string `json:"selector,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:shortName={eg}
+// +kubebuilder:subresource:scale:selectorpath=.status.selector,specpath=.spec.replicas,statuspath=.status.replicas
+
+// Egress is the Schema for the egresses API
+type Egress struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   EgressSpec   `json:"spec,omitempty"`
+	Status EgressStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// EgressList contains a list of Egress
+type EgressList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Egress `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Egress{}, &EgressList{})
+}

api/v1beta1/groupversion_info.go

@@ -0,0 +1,20 @@
+// Package v1beta1 contains API Schema definitions for the pona v1beta1 API group
+// +kubebuilder:object:generate=true
+// +groupName=pona.cybozu.com
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "pona.cybozu.com", Version: "v1beta1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)