chore: e2e & bug fix & update readme

README.md

@@ -16,7 +16,7 @@ The design architecture of this project is based on [openkruise/controllermesh](
 3. **Circuit breaker and rate limiter**: Not only Kubernetes operation requests, but also other external operation requests.
 4. **Multicluster routing and sharding**: This feature is supported by [kusionstack/kaera(karbour)]()
 
-<p align="center"><img width="800" src="./docs/img/img2.png"/></p>
+<p align="center"><img width="800" src="./docs/img/img4.png"/></p>
 
 ## Quick Start
 Visit [Quick Start]().
@@ -71,28 +71,27 @@ spec:
     certDir: /tmp/webhook-certs
     port: 9443
   limits:
-    - objectSelector:
-        relateResources:
-          - apiGroups:
-              - '*'
-            resources:
-              - pods
-              - services
-        selector:
-          matchExpressions:
-            - key: kridge.kusionstack.io/namespace
-              operator: In
-              values:
-                - ns-a
-                - ns-b
-          matchLabels:
-          # ...
-  selector:
-    matchExpressions:
-      - key: statefulset.kubernetes.io/pod-name
+  - relateResources:
+    - apiGroups:
+      - '*'
+      resources:
+      - pods
+      - services
+    selector:
+      matchExpressions:
+      - key: kridge.kusionstack.io/namespace
         operator: In
         values:
-          - operator-demo-0
+        - ns-a
+        - ns-b
+      matchLabels:
+      # ...
+  selector:
+    matchExpressions:
+    - key: statefulset.kubernetes.io/pod-name
+      operator: In
+      values:
+      - operator-demo-0
 ```
 
 - selector: for all pods under a shard. It can be a subset of pods under a StatefulSet.

artifacts/images/custom.Dockerfile

@@ -0,0 +1,35 @@
+# Build the manager binary
+FROM golang:1.19 as builder
+
+WORKDIR /workspace
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY e2e/ e2e/
+COPY vendor/ vendor/
+
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 go build -mod=vendor -a -o testapp ./e2e/customoperator/app/main.go
+
+
+FROM ubuntu:focal
+
+
+RUN apt-get update && \
+  apt-get install --no-install-recommends -y \
+  ca-certificates \
+  curl \
+  iputils-ping \
+  tcpdump \
+  iproute2 \
+  iptables \
+  net-tools \
+  telnet \
+  lsof \
+  linux-tools-generic \
+  sudo && \
+  apt-get clean && \
+  rm -rf  /var/log/*log /var/lib/apt/lists/* /var/log/apt/* /var/lib/dpkg/*-old /var/cache/debconf/*-old
+
+WORKDIR /
+COPY --from=builder /workspace/testapp .
+ENTRYPOINT ["/testapp"]

artifacts/images/manager.Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.19 as builder
+FROM golang:1.20 as builder
 
 WORKDIR /workspace
 

artifacts/images/proxy.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19 as builder
+FROM golang:1.20 as builder
 
 WORKDIR /workspace
 
@@ -8,7 +8,7 @@ COPY artifacts/ artifacts/
 COPY pkg/ pkg/
 COPY vendor/ vendor/
 
-RUN CGO_ENABLED=0 GO111MODULE=on go build -mod=vendor -a -o kridge-proxy ./pkg/cmd/proxy/main.go
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 go build -mod=vendor -a -o kridge-proxy ./pkg/cmd/proxy/main.go
 
 FROM ubuntu:focal
 
@@ -33,6 +33,6 @@ RUN useradd -m --uid 1359 kridge-proxy && \
   echo "kridge-proxy ALL=NOPASSWD: ALL" >> /etc/sudoers
 WORKDIR /
 COPY artifacts/scripts/proxy-poststart.sh /poststart.sh
-
+RUN mkdir /kridge && chmod 777 /kridge
 COPY --from=builder /workspace/kridge-proxy .
 ENTRYPOINT ["/kridge-proxy"]

docs/installation.md

@@ -0,0 +1,18 @@
+
+## Installation
+
+**Install by helm**
+```shell
+# Firstly add charts repository if you haven't do this.
+$ helm repo add kusionstack https://kusionstack.github.io/charts/
+
+# [Optional]
+$ helm repo update
+
+# Install the latest version.
+$ helm install kridge kusionstack/kridge --version 0.0.x
+
+# Uninstall
+$ helm uninstall kridge
+```
+

docs/intro.md

@@ -0,0 +1,16 @@
+
+
+# Kridge
+
+Kridge is a solution that helps developers manage their controllers/operators better.
+
+The design architecture of this project is based on [openkruise/controllermesh](https://github.com/openkruise/controllermesh).
+
+## Key Features
+
+1. **Sharding**: Through relevant configurations, Kubernetes single-point deployed operator applications can be flexibly shard deployed.
+2. **Canary upgrade**: Depends on sharding, the controllers can be updated in canary progress instead of one time replace.
+3. **Circuit breaker and rate limiter**: Not only Kubernetes operation requests, but also other external operation requests.
+4. **Multicluster routing and sharding**: This feature is supported by [kusionstack/kaera(karbour)]()
+
+<p align="center"><img width="800" src="../docs/img/img4.png"/></p>

e2e/customoperator/app/controller/pod_recorder.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2023 The KusionStack Authors.
+Modified from Kruise code, Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"os"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+var (
+	podName, podNamespace string
+)
+
+// ManagerStateReconciler reconciles a ManagerState object
+type PodReconciler struct {
+	client.Client
+	DirectorClient client.Client
+}
+
+func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, err error) {
+	return reconcile.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	podNamespace = os.Getenv("POD_NAMESPACE")
+	podName = os.Getenv("POD_NAME")
+	// default handler.EnqueueRequestForObject
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1.Pod{}).
+		Watches(&source.Kind{Type: &v1.Pod{}}, &enqueueHandler{Client: r.DirectorClient, kind: "Pod"}).
+		Watches(&source.Kind{Type: &v1.ConfigMap{}}, &enqueueHandler{Client: r.DirectorClient, kind: "ConfigMap"}).
+		Complete(r)
+}
+
+type enqueueHandler struct {
+	client.Client
+	kind string
+}
+
+func (e *enqueueHandler) Create(event event.CreateEvent, q workqueue.RateLimitingInterface) {
+	Add(e.Client, event.Object.GetNamespace(), event.Object.GetName(), e.kind)
+}
+
+func (e *enqueueHandler) Update(event event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	Add(e.Client, event.ObjectNew.GetNamespace(), event.ObjectNew.GetName(), e.kind)
+}
+
+func (e *enqueueHandler) Delete(event event.DeleteEvent, q workqueue.RateLimitingInterface) {
+	Add(e.Client, event.Object.GetNamespace(), event.Object.GetName(), e.kind)
+}
+
+func (e *enqueueHandler) Generic(event event.GenericEvent, q workqueue.RateLimitingInterface) {
+	Add(e.Client, event.Object.GetNamespace(), event.Object.GetName(), e.kind)
+}
+
+func Add(c client.Client, namespace, name, kind string) {
+	klog.Infof("handle event %s %s/%s", kind, namespace, name)
+	if err := add(c, namespace, kind); err != nil {
+		klog.Errorf("fail to record event %s %s/%s, %v", kind, namespace, name, err)
+	}
+}

e2e/customoperator/app/controller/util.go

@@ -0,0 +1,143 @@
+package controller
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	cmName = "test-resource-recorder"
+)
+
+func initConfigMap(c client.Client) *v1.ConfigMap {
+	//podName := os.Getenv("POD_NAME")
+	cm := &v1.ConfigMap{}
+	cm.Name = cmName
+	cm.Namespace = podNamespace
+	cm.Data = map[string]string{}
+	if err := c.Create(context.TODO(), cm); err != nil {
+		klog.Errorf("Failed to create cm %v", err)
+	}
+	return cm
+}
+
+var Retry = wait.Backoff{
+	Steps:    50,
+	Duration: 10 * time.Millisecond,
+	Factor:   1.0,
+	Jitter:   0.1,
+}
+
+func add(c client.Client, namespace, kind string) error {
+	if namespace == "" || kind == "" {
+		return nil
+	}
+	key := podName
+	return retry.RetryOnConflict(Retry, func() error {
+		cm := &v1.ConfigMap{}
+		if err := c.Get(context.TODO(), types.NamespacedName{Name: cmName, Namespace: podNamespace}, cm); err != nil {
+			klog.Errorf("Failed to get cm %v, try init", err)
+			cm = initConfigMap(c)
+		}
+		if cm.Data == nil {
+			cm.Data = map[string]string{}
+		}
+		val, _ := cm.Data[key]
+		sto := &store{}
+		if val == "" {
+			sto.Kinds = map[string]string{}
+		} else {
+			json.Unmarshal([]byte(val), sto)
+		}
+		names, _ := sto.Kinds[kind]
+		nameSet := list(names)
+		if nameSet.Has(namespace) {
+			return nil
+		}
+		nameSet.Insert(namespace)
+		sto.Kinds[kind] = toString(nameSet)
+		bt, _ := json.Marshal(sto)
+		cm.Data[key] = string(bt)
+		return c.Update(context.TODO(), cm)
+	})
+}
+
+func Checker(ctx context.Context, c client.Client) {
+	cm := &v1.ConfigMap{}
+	for {
+		resources := map[string]sets.Set[string]{}
+		<-time.After(20 * time.Second)
+
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if err := c.Get(context.TODO(), types.NamespacedName{Name: cmName, Namespace: podNamespace}, cm); err != nil {
+			klog.Errorf("fail to get configMap %s, %v", cmName, err)
+			continue
+		}
+		if cm.Data == nil {
+			klog.Infof("nil configmap data")
+			continue
+		}
+		for pod, val := range cm.Data {
+			sto := &store{}
+			if err := json.Unmarshal([]byte(val), sto); err != nil {
+				klog.Errorf("fail to unmarshal store, %v", err)
+			}
+			if sto.Kinds == nil {
+				continue
+			}
+			for kind, names := range sto.Kinds {
+				set, ok := resources[kind]
+				if !ok {
+					set = list(names)
+					resources[kind] = set
+					continue
+				}
+				nameList := list(names).UnsortedList()
+				for _, name := range nameList {
+					if set.Has(name) {
+						msg := fmt.Sprintf("conflict namespace %s in store, pod: %s", name, pod)
+						klog.Errorf(msg)
+						<-time.After(20 * time.Second)
+						panic(msg)
+					}
+					set.Insert(name)
+				}
+				resources[kind] = set
+			}
+		}
+	}
+}
+
+func list(val string) sets.Set[string] {
+	res := sets.New[string]()
+	if val == "" {
+		return res
+	}
+
+	res.Insert(strings.Split(val, ",")...)
+	return res
+}
+
+func toString(names sets.Set[string]) string {
+	return strings.Join(sets.List(names), ",")
+}
+
+type store struct {
+	Kinds map[string]string
+}