implement grpc metrics for ceph-csi

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
ceph · Aug 30, 2019 · a81a3bf · a81a3bf
1 parent 01a78ca
commit a81a3bf
Show file tree

Hide file tree

Showing 46 changed files with 1,362 additions and 157 deletions.
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Gopkg.toml b/Gopkg.toml
@@ -59,6 +59,14 @@
   name = "k8s.io/kube-aggregator"
   version = "kubernetes-1.15.2"
 
+[[constraint]]
+  name = "github.com/grpc-ecosystem/go-grpc-prometheus"
+  version = "1.2.0"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "1.1.0"
+
 [prune]
   go-tests = true
   non-go = true

diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go
@@ -65,11 +65,16 @@ func init() {
 	flag.StringVar(&conf.VolumeMounter, "volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
 	flag.StringVar(&conf.MountCacheDir, "mountcachedir", "", "mount info cache save dir")
 
-	// livenes related flags
-	flag.IntVar(&conf.LivenessPort, "livenessport", 8080, "TCP port for liveness requests")
-	flag.StringVar(&conf.LivenessPath, "livenesspath", "/metrics", "path of prometheus endpoint where metrics will be available")
+	// liveness/grpc metrics related flags
+	flag.IntVar(&conf.MetricsPort, "metricsport", 8080, "TCP port for liveness/grpc metrics requests")
+	flag.StringVar(&conf.MetricsPath, "metricspath", "/metrics", "path of prometheus endpoint where metrics will be available")
 	flag.DurationVar(&conf.PollTime, "polltime", time.Second*60, "time interval in seconds between each poll")
 	flag.DurationVar(&conf.PoolTimeout, "timeout", time.Second*3, "probe timeout in seconds")
+
+	flag.BoolVar(&conf.EnableGRPCMetrics, "enablegrpcmetrics", false, "enable grpc metrics")
+	flag.StringVar(&conf.HistogramOption, "histogramoption", "0.5,2,6",
+		"Histogram option for grpc metrics, should be comma separated value, ex:= 0.5,2,6 where start=0.5 factor=2, count=6")
+
 	klog.InitFlags(nil)
 	if err := flag.Set("logtostderr", "true"); err != nil {
 		klog.Exitf("failed to set logtostderr flag: %v", err)
@@ -119,9 +124,9 @@ func main() {
 
 	// the driver may need a higher PID limit for handling all concurrent requests
 	if conf.PidLimit != 0 {
-		currentLimit, err := util.GetPIDLimit()
-		if err != nil {
-			klog.Errorf("Failed to get the PID limit, can not reconfigure: %v", err)
+		currentLimit, pidErr := util.GetPIDLimit()
+		if pidErr != nil {
+			klog.Errorf("Failed to get the PID limit, can not reconfigure: %v", pidErr)
 		} else {
 			klog.Infof("Initial PID limit is set to %d", currentLimit)
 			err = util.SetPIDLimit(conf.PidLimit)
@@ -137,6 +142,20 @@ func main() {
 		}
 	}
 
+	if conf.EnableGRPCMetrics || conf.Vtype == livenessType {
+		// validate metrics endpoint
+		conf.MetricsIP = os.Getenv("POD_IP")
+
+		if conf.MetricsIP == "" {
+			klog.Warning("missing POD_IP env var defaulting to 0.0.0.0")
+			conf.MetricsIP = "0.0.0.0"
+		}
+		err = util.ValidateURL(&conf)
+		if err != nil {
+			klog.Fatalln(err)
+		}
+	}
+
 	klog.Infof("Starting driver type: %v with name: %v", conf.Vtype, dname)
 	switch conf.Vtype {
 	case rbdType:

diff --git a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml
@@ -4,7 +4,7 @@ apiVersion: v1
 metadata:
   name: csi-cephfsplugin-provisioner
   labels:
-    app: csi-liveness
+    app: csi-metrics
 spec:
   selector:
     app: csi-cephfsplugin-provisioner
@@ -13,6 +13,10 @@ spec:
       port: 8080
       protocol: TCP
       targetPort: 8081
+    - name: grpc-metrics
+      port: 8090
+      protocol: TCP
+      targetPort: 8091
 
 ---
 kind: StatefulSet
@@ -74,7 +78,14 @@ spec:
             - "--drivername=cephfs.csi.ceph.com"
             - "--metadatastorage=k8s_configmap"
             - "--pidlimit=-1"
+            - "--metricsport=8091"
+            - "--metricspath=/metrics"
+            - "--enablegrpcmetrics=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: NODE_ID
               valueFrom:
                 fieldRef:
@@ -105,8 +116,8 @@ spec:
           args:
             - "--type=liveness"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--livenessport=8081"
-            - "--livenesspath=/metrics"
+            - "--metricsport=8081"
+            - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
           env:

diff --git a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml
@@ -59,7 +59,14 @@ spec:
             - "--drivername=cephfs.csi.ceph.com"
             - "--metadatastorage=k8s_configmap"
             - "--mountcachedir=/mount-cache-dir"
+            - "--metricsport=8091"
+            - "--metricspath=/metrics"
+            - "--enablegrpcmetrics=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: NODE_ID
               valueFrom:
                 fieldRef:
@@ -99,8 +106,8 @@ spec:
           args:
             - "--type=liveness"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--livenessport=8081"
-            - "--livenesspath=/metrics"
+            - "--metricsport=8081"
+            - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
           env:
@@ -150,18 +157,22 @@ spec:
             medium: "Memory"
           }
 ---
-# This is a service to expose the liveness side car
+# This is a service to expose the liveness and grpc metrics
 apiVersion: v1
 kind: Service
 metadata:
-  name: csi-liveness-cephfsplugin
+  name: csi-metrics-cephfsplugin
   labels:
-    app: csi-liveness
+    app: csi-metrics
 spec:
   ports:
     - name: http-metrics
       port: 8080
       protocol: TCP
       targetPort: 8081
+    - name: grpc-metrics
+      port: 8090
+      protocol: TCP
+      targetPort: 8091
   selector:
     app: csi-cephfsplugin
diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml
@@ -74,7 +74,14 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--metadatastorage=k8s_configmap"
             - "--mountcachedir=/mount-cache-dir"
+            - "--metricsport=8091"
+            - "--metricspath=/metrics"
+            - "--enablegrpcmetrics=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID
@@ -115,8 +122,8 @@ spec:
           args:
             - "--type=liveness"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--livenessport=8081"
-            - "--livenesspath=/metrics"
+            - "--metricsport=8081"
+            - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
           imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}

diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml
@@ -0,0 +1,25 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: csi-metrics
+    chart: {{ include "ceph-csi-cephfs.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    app: {{ include "ceph-csi-cephfs.name" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+  ports:
+    - name: http-metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8081
+    - name: grpc-metrics
+      port: 8090
+      protocol: TCP
+      targetPort: 8091
diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-service.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-service.yaml
@@ -4,7 +4,7 @@ metadata:
   name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
   namespace: {{ .Release.Namespace }}
   labels:
-    app: csi-liveness
+    app: csi-metrics
     chart: {{ include "ceph-csi-cephfs.chart" . }}
     component: {{ .Values.provisioner.name }}
     release: {{ .Release.Name }}
@@ -19,3 +19,7 @@ spec:
       port: 8080
       protocol: TCP
       targetPort: 8081
+    - name: grpc-metrics
+      port: 8090
+      protocol: TCP
+      targetPort: 8091
diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml
@@ -77,7 +77,14 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
             - "--metadatastorage=k8s_configmap"
             - "--pidlimit=-1"
+            - "--metricsport=8091"
+            - "--metricspath=/metrics"
+            - "--enablegrpcmetrics=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: DRIVER_NAME
               value: {{ .Values.driverName }}
             - name: NODE_ID
@@ -102,8 +109,8 @@ spec:
           args:
             - "--type=liveness"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--livenessport=8081"
-            - "--livenesspath=/metrics"
+            - "--metricsport=8081"
+            - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
           imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}

diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml
@@ -4,7 +4,7 @@ apiVersion: v1
 metadata:
   name: csi-cephfsplugin-provisioner
   labels:
-    app: csi-liveness
+    app: csi-metrics
 spec:
   selector:
     app: csi-cephfsplugin-provisioner
@@ -13,6 +13,10 @@ spec:
       port: 8080
       protocol: TCP
       targetPort: 8081
+    - name: grpc-metrics
+      port: 8090
+      protocol: TCP
+      targetPort: 8091
 
 ---
 kind: Deployment
@@ -77,7 +81,14 @@ spec:
             - "--drivername=cephfs.csi.ceph.com"
             - "--metadatastorage=k8s_configmap"
             - "--pidlimit=-1"
+            - "--metricsport=8091"
+            - "--metricspath=/metrics"
+            - "--enablegrpcmetrics=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: NODE_ID
               valueFrom:
                 fieldRef:
@@ -108,8 +119,8 @@ spec:
           args:
             - "--type=liveness"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--livenessport=8081"
-            - "--livenesspath=/metrics"
+            - "--metricsport=8081"
+            - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
           env:

diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml
@@ -59,7 +59,14 @@ spec:
             - "--drivername=cephfs.csi.ceph.com"
             - "--metadatastorage=k8s_configmap"
             - "--mountcachedir=/mount-cache-dir"
+            - "--metricsport=8090"
+            - "--metricspath=/metrics"
+            - "--enablegrpcmetrics=false"
           env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: NODE_ID
               valueFrom:
                 fieldRef:
@@ -98,8 +105,8 @@ spec:
           args:
             - "--type=liveness"
             - "--endpoint=$(CSI_ENDPOINT)"
-            - "--livenessport=8081"
-            - "--livenesspath=/metrics"
+            - "--metricsport=8081"
+            - "--metricspath=/metrics"
             - "--polltime=60s"
             - "--timeout=3s"
           env:
@@ -149,18 +156,22 @@ spec:
             medium: "Memory"
           }
 ---
-# This is a service to expose the liveness side car
+# This is a service to expose the liveness and grpc metrics
 apiVersion: v1
 kind: Service
 metadata:
-  name: csi-liveness-cephfsplugin
+  name: csi-metrics-cephfsplugin
   labels:
-    app: csi-liveness
+    app: csi-metrics
 spec:
   ports:
     - name: http-metrics
       port: 8080
       protocol: TCP
       targetPort: 8081
+    - name: grpc-metrics
+      port: 8090
+      protocol: TCP
+      targetPort: 8091
   selector:
     app: csi-cephfsplugin