modify store and proxy startup script and discovery svc

Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:alpine
+
+RUN apk update; apk add curl netcat-openbsd
+
+COPY . /root
+
+WORKDIR /root
+RUN GO111MODULE=on; go build -o /usr/local/bin/cell-controller-manager -mod=vendor ./cmd/controller-manager/main.go
+RUN GO111MODULE=on; go build -o /usr/local/bin/cell-discovery          -mod=vendor ./cmd/discovery/main.go
+RUN GO111MODULE=on; go build -o /usr/local/bin/cell-scheduler          -mod=vendor ./cmd/scheduler/main.go
+
+RUN mv etcdctl /usr/local/bin

charts/cell-cluster/templates/cell-service.yaml

@@ -22,21 +22,11 @@ spec:
   {{- end }}
   ports:
   - name: redis-client
-    port: 4000
-    targetPort: 4000
+    port: 6379
+    targetPort: 6379
     protocol: TCP
-    {{- if .Values.proxy.service.mysqlNodePort }}
-    nodePort: {{ .Values.proxy.service.mysqlNodePort }}
-    {{- end }}
-  {{- if .Values.proxy.service.exposeStatus }}
-  - name: status
-    port: 10080
-    targetPort: 10080
-    protocol: TCP
-    {{- if .Values.proxy.service.statusNodePort }}
-    nodePort: {{ .Values.proxy.service.statusNodePort }}
-    {{- end }}
-  {{- end }}
+
+
   selector:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}

charts/cell-cluster/templates/discovery-deployment.yaml

@@ -46,5 +46,5 @@ spec:
               fieldRef:
                 fieldPath: metadata.namespace
           - name: CLUSTER_NAME
-            value: {{ .Release.Name }}
+            value: {{ .Values.clusterName }}
 

charts/cell-cluster/templates/scripts/_start_pd.sh.tpl

@@ -27,11 +27,13 @@ fi
 
 # the general form of variable PEER_SERVICE_NAME is: "<clusterName>-pd"
 cluster_name=`echo ${SERVICE_NAME} | sed 's/-pd//'`
-domain="${HOSTNAME}.${SERVICE_NAME}.${NAMESPACE}.svc"
+hostip=`hostname -i | tr "\n" " " | sed "s/ //g"`
+domain="${HOSTNAME}.${SERVICE_NAME}.${NAMESPACE}"
 pd_domain="${SERVICE_NAME}.${NAMESPACE}.svc"
-discovery_url="${cluster_name}-discovery.${NAMESPACE}.svc:10261"
-encoded_domain_url=`echo ${domain}:2379 | base64 | tr "\n" " " | sed "s/ //g"`
 
+discovery_url="${cluster_name}-discovery.${NAMESPACE}.svc:10261"
+encoded_domain_url=`echo ${domain}  | tr "\n" " " | sed "s/ //g" | base64`
+encoded_pod_ip=`echo ${hostip} | tr "\n" " " | sed "s/ //g" | base64`
 elapseTime=0
 period=1
 threshold=30
@@ -47,28 +49,31 @@ while true; do
 
     if nslookup ${pd_domain} 2>/dev/null
     then
-        echo "nslookup domain ${pd_domain}.svc success"
+        echo "nslookup domain ${pd_domain} success"
         break
     else
         echo "nslookup domain ${pd_domain} failed" >&2
     fi
+
 done
 
+
 ARGS="--data=/var/lib/pd/data \
---log-file=/var/lib/pd/pd.log \
+-log-level=debug \
 --name=${HOSTNAME} \
---addr-rpc=:20800 \
---urls-client=http://0.0.0.0:2379 \
---urls-peer=http://0.0.0.0:2380 \
---initial-cluster=
+--addr-rpc=${hostip}:20800 \
+--urls-client=http://${hostip}:2379 \
+--urls-peer=http://${hostip}:2380 \
+--initial-cluster=\
 "
-until result=$(wget -qO- -T 3 http://${discovery_url}/new/${encoded_domain_url} 2>/dev/null); do
+
+until result=$(wget -qO- -T 3 http://${discovery_url}/new/${encoded_domain_url}/${encoded_pod_ip} 2>/dev/null); do
     echo "waiting for discovery service returns start args ..."
     sleep $((RANDOM % 5))
 done
 ARGS="${ARGS}${result}"
 
 echo "starting pd-server ..."
 sleep $((RANDOM % 10))
-echo "/usr/local/bin ${ARGS}"
-exec /usr/local/bin ${ARGS}
+echo "/usr/local/bin/pd ${ARGS}"
+exec /usr/local/bin/pd ${ARGS}

charts/cell-cluster/templates/scripts/_start_proxy.sh.tpl

@@ -24,19 +24,19 @@ then
 fi
 
 # the general form of variable PEER_SERVICE_NAME is: "<clusterName>-pd-peer"
-cluster_name=${CLUSTER_NAME}
-discovery_url="${cluster_name}-discovery.${NAMESPACE}.svc:10261"
-
-ARGS="
---cfg=/etc/proxy/proxy-config.json
-"
+discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}.svc:10261"
 
 until result=$(wget -qO- -T 3 http://${discovery_url}/proxy-config 2>/dev/null); do
     echo "waiting for discovery service returns start args ..."
     sleep $((RANDOM % 5))
 done
-echo ${resutl} > /etc/proxy/proxy-config.json
+echo ${result} > /var/lib/redis-proxy/proxy-config.json
+
+ARGS="\
+-log-level=debug \
+--cfg=/var/lib/redis-proxy/proxy-config.json \
+"
 
 echo "start cell-proxy ..."
 cat /etc/proxy/proxy-config.json
-exec /redis-proxy ${ARGS}
+exec /usr/local/bin/redis-proxy ${ARGS}

charts/cell-cluster/templates/scripts/_start_store.sh.tpl

@@ -25,14 +25,22 @@ then
 	tail -f /dev/null
 fi
 
+discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}.svc:10261"
+
+until result=$(wget -qO- -T 3 http://${discovery_url}/store-config 2>/dev/null); do
+    echo "waiting for discovery service returns start args ..."
+    sleep $((RANDOM % 5))
+done
+
+hostip=`hostname -i | tr "\n" " " | sed "s/ //g"`
 ARGS="--data=/var/lib/cell/data \
---log-file=/var/lib/cell/cell.log \
---addr=${HOSTNAME}:10800 \
+-log-level=debug \
+--addr=${hostip}:10800 \
 --addr-cli=:6370 \
 --zone=zone-1 --rack=rack-1 \
 --interval-heartbeat-store=5 \
 --interval-heartbeat-cell=2 \
---pd=${CLUSTER_NAME}-pd:2379
+--pd=${result}\
 "
 
 echo "starting cell ..."

charts/cell-cluster/values.yaml

@@ -26,7 +26,7 @@ services:
 
 discovery:
   image: harbor.infinivision.cn/deepfabric/elasticell-operator:latest
-  imagePullPolicy: IfNotPresent
+  imagePullPolicy: Always
   resources:
     limits:
       cpu: 250m
@@ -46,7 +46,7 @@ pd:
   storageClassName: local-storage
 
   # Image pull policy.
-  imagePullPolicy: IfNotPresent
+  imagePullPolicy: Always
 
   # maxStoreDownTime is how long a store will be considered `down` when disconnected
   # if a store is considered `down`, the regions will be migrated to other stores
@@ -86,7 +86,7 @@ store:
   storageClassName: local-storage
 
   # Image pull policy.
-  imagePullPolicy: IfNotPresent
+  imagePullPolicy: Always
 
   # syncLog is a bool value to enable or disable syc-log for raftstore, default is true
   # enable this can prevent data loss when power failure
@@ -130,7 +130,7 @@ proxy:
   replicas: 2
   image: harbor.infinivision.cn/deepfabric/redis-proxy:latest
   # Image pull policy.
-  imagePullPolicy: IfNotPresent
+  imagePullPolicy: Always
   logLevel: info
   resources:
     limits: {}

charts/cell-operator/values.yaml

@@ -10,7 +10,7 @@ rbac:
 
 # operatorImage is Elasticell Operator image
 operatorImage: harbor.infinivision.cn/deepfabric/elasticell-operator:latest
-imagePullPolicy: IfNotPresent
+imagePullPolicy: Always
 
 defaultStorageClassName: local-storage
 
@@ -19,7 +19,7 @@ controllerManager:
   # With rbac.create=true, this service account will be created
   # Also see rbac.create and clusterScoped
   serviceAccount: cell-controller-manager
-  logLevel: 2
+  logLevel: 6
   replicas: 1
   resources:
     limits:

cmd/controller-manager/main.go

@@ -113,14 +113,14 @@ func main() {
 		},
 	}
 
-	tcController := cellcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
+	ccController := cellcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
 	stop := make(chan struct{})
 	defer close(stop)
 	go informerFactory.Start(stop)
 	go kubeInformerFactory.Start(stop)
 
 	onStarted := func(stopCh <-chan struct{}) {
-		tcController.Run(workers, stopCh)
+		ccController.Run(workers, stopCh)
 	}
 	onStopped := func() {
 		glog.Fatalf("leader election lost")

go.mod

@@ -37,9 +37,10 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
-	github.com/onsi/ginkgo v1.6.0
+	github.com/onsi/ginkgo v1.6.0 // indirect
 	github.com/onsi/gomega v1.4.1
 	github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 // indirect
+	github.com/pingcap/errors v0.11.1
 	github.com/pkg/errors v0.8.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v0.8.0 // indirect
@@ -59,9 +60,9 @@ require (
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1 // indirect
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a // indirect
 	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
-	golang.org/x/tools v0.0.0-20190328030505-8f05a32dce9f // indirect
 	google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5 // indirect
 	google.golang.org/grpc v1.12.0 // indirect
 	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect

go.sum

@@ -76,6 +76,8 @@ github.com/onsi/gomega v1.4.1 h1:PZSj/UFNaVp3KxrzHOcS7oyuWA7LoOY/77yCTEFu21U=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ=
 github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
+github.com/pingcap/errors v0.11.1 h1:BXFZ6MdDd2U1uJUa2sRAWTmm+nieEzuyYM0R4aUTcC8=
+github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -126,8 +128,6 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20190328030505-8f05a32dce9f h1:MC9ePo9/ITLPbt6QsoTD78Pg07by0QbfGLUQpkJ01QM=
-golang.org/x/tools v0.0.0-20190328030505-8f05a32dce9f/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5 h1:2PjFmwzH/sxgW9CRJDlEiwMHO8rOk1eMDzVL14HC1e4=
 google.golang.org/genproto v0.0.0-20180731170733-daca94659cb5/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.12.0 h1:Mm8atZtkT+P6R43n/dqNDWkPPu5BwRVu/1rJnJCeZH8=

pkg/apis/deepfabric.com/v1alpha1/types.go

@@ -91,13 +91,15 @@ type CellClusterSpec struct {
 	Services        []Service                            `json:"services,omitempty"`
 	PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
 	Timezone        string                               `json:"timezone,omitempty"`
+
+	PdPeerURL string `json:"peerURL,omitempty"`
+	PdPeerRPC string `json:"peerRPC,omitempty"`
 }
 
 // CellClusterStatus represents the current status of a cell cluster.
 type CellClusterStatus struct {
 	ClusterID string      `json:"clusterID,omitempty"`
 	PD        PDStatus    `json:"pd,omitempty"`
-	PdPeerURL string      `json:"peerurl,omitempty"`
 	Store     StoreStatus `json:"store,omitempty"`
 	Proxy     ProxyStatus `json:"proxy,omitempty"`
 }

pkg/controller/cellcluster/cell_cluster_control.go

@@ -18,6 +18,7 @@ import (
 	"github.com/deepfabric/elasticell-operator/pkg/controller"
 	"github.com/deepfabric/elasticell-operator/pkg/manager"
 	"github.com/deepfabric/elasticell-operator/pkg/manager/member"
+	"github.com/golang/glog"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/tools/record"
@@ -86,11 +87,13 @@ func (ccc *defaultCellClusterControl) UpdateCellCluster(cc *v1alpha1.CellCluster
 func (ccc *defaultCellClusterControl) updateCellCluster(cc *v1alpha1.CellCluster) error {
 	// syncing all PVs managed by operator's reclaim policy to Retain
 	if err := ccc.reclaimPolicyManager.Sync(cc); err != nil {
+		glog.Errorf("reclaimPolicyManager failed")
 		return err
 	}
 
 	// cleaning all orphan pods(pd or store which don't have a related PVC) managed by operator
 	if _, err := ccc.orphanPodsCleaner.Clean(cc); err != nil {
+		glog.Errorf("orphanPodsCleaner failed")
 		return err
 	}
 
@@ -106,6 +109,7 @@ func (ccc *defaultCellClusterControl) updateCellCluster(cc *v1alpha1.CellCluster
 	//   - scale out/in the pd cluster
 	//   - failover the pd cluster
 	if err := ccc.pdMemberManager.Sync(cc); err != nil {
+		glog.Errorf("pdMemberManager failed")
 		return err
 	}
 
@@ -119,6 +123,7 @@ func (ccc *defaultCellClusterControl) updateCellCluster(cc *v1alpha1.CellCluster
 	//   - scale out/in the store cluster
 	//   - failover the store cluster
 	if err := ccc.storeMemberManager.Sync(cc); err != nil {
+		glog.Errorf("storeMemberManager failed")
 		return err
 	}
 
@@ -131,12 +136,19 @@ func (ccc *defaultCellClusterControl) updateCellCluster(cc *v1alpha1.CellCluster
 	//   - scale out/in the proxy cluster
 	//   - failover the proxy cluster
 	if err := ccc.proxyMemberManager.Sync(cc); err != nil {
+		glog.Errorf("proxyMemberManager failed")
 		return err
 	}
 
 	// syncing the labels from Pod to PVC and PV, these labels include:
 	//   - label.StoreIDLabelKey
 	//   - label.MemberIDLabelKey
 	//   - label.NamespaceLabelKey
-	return ccc.metaManager.Sync(cc)
+
+	if err := ccc.metaManager.Sync(cc); err != nil {
+		glog.Errorf("metaManager failed")
+		return err
+	}
+
+	return nil
 }

pkg/controller/cellcluster/cell_cluster_controller.go

@@ -25,6 +25,7 @@ import (
 	mm "github.com/deepfabric/elasticell-operator/pkg/manager/member"
 	"github.com/deepfabric/elasticell-operator/pkg/manager/meta"
 	"github.com/golang/glog"
+	perrors "github.com/pingcap/errors"
 	apps "k8s.io/api/apps/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -209,7 +210,9 @@ func (ccc *Controller) Run(workers int, stopCh <-chan struct{}) {
 func (ccc *Controller) worker() {
 	for ccc.processNextWorkItem() {
 		// revive:disable:empty-block
+		glog.V(2).Infof("process one item")
 	}
+	glog.V(2).Infof("worker over")
 }
 
 // processNextWorkItem dequeues items, processes them, and marks them done. It enforces that the syncHandler is never
@@ -220,15 +223,13 @@ func (ccc *Controller) processNextWorkItem() bool {
 		return false
 	}
 	defer ccc.queue.Done(key)
+	glog.V(2).Infof("do sync")
 	if err := ccc.sync(key.(string)); err != nil {
-		/*
-			// to do
-			if perrors.Find(err, controller.IsRequeueError) != nil {
-				glog.Infof("CellCluster: %v, still need sync: %v, requeuing", key.(string), err)
-			} else {
-				utilruntime.HandleError(fmt.Errorf("CellCluster: %v, sync failed %v, requeuing", key.(string), err))
-			}
-		*/
+		if perrors.Find(err, controller.IsRequeueError) != nil {
+			glog.Infof("CellCluster: %v, still need sync: %v, requeuing", key.(string), err)
+		} else {
+			utilruntime.HandleError(fmt.Errorf("CellCluster: %v, sync failed %v, requeuing", key.(string), err))
+		}
 		ccc.queue.AddRateLimited(key)
 	} else {
 		ccc.queue.Forget(key)