使用 Blackbox exporter 监控 k8s service

[Bpazy]

blackbox k8s 配置:

apiVersion: v1
kind: ConfigMap
metadata:
  name: blackbox-exporter
  namespace: monitoring
data:
  blackbox.yml: |
    modules:
      http_2xx:
        prober: http
        http:
          method: GET
          preferred_ip_protocol: ipv4

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: blackbox-exporter
  namespace: monitoring
  labels:
    name: blackbox-exporter
  annotations:
    reloader.stakater.com/auto: "true"
spec:
  replicas: 1
  selector:
    matchLabels:
      name: blackbox-exporter
  template:
    metadata:
      labels:
        name: blackbox-exporter
    spec:
      containers:
      - name: blackbox-exporter
        image: quay.io/prometheus/blackbox-exporter:latest
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        ports:
        - containerPort: 9115
          name: http
        args:
        - "--config.file=/etc/prometheus/blackbox.yml"
        volumeMounts:
        - name: blackbox-exporter-config
          mountPath: /etc/prometheus/
          readOnly: true
      volumes:
      - name: blackbox-exporter-config
        configMap:
          name: blackbox-exporter

---
apiVersion: v1
kind: Service
metadata:
  name: blackbox-exporter
  namespace: monitoring
spec:
  type: ClusterIP
  selector:
    name: blackbox-exporter
  ports:
  - port: 9115
    targetPort: 9115

```yaml
核心是 prometheus 的配置:
```yaml
      # 这里监控外部网站
      - job_name: 'blackbox'
        metrics_path: /probe
        params:
          module: [http_2xx]
        static_configs:
          - targets:
            - https://baidu.com
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: 'blackbox-exporter:9115'

      # 这里监控 k8s service 服务
      - job_name: 'blackbox_service'
        metrics_path: /probe
        params:
          module: [http_2xx]
        scheme: https
        api_server: https://ziyuan360.host:6443
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: service
        relabel_configs:
        - action: keep
          source_labels: [__meta_kubernetes_service_annotation_prometheus_io_http_probe]
          regex: "true"
        - action: replace
          source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_http_probe_path]
          target_label: __param_target
          regex: (.+);(.+)
          replacement: $1$2
        - source_labels: [__param_target]
          target_label: target
        - action: replace
          source_labels: [__address__]
          target_label: instance
          regex: (.+?)\..+
          replacement: $1
        - target_label: __scheme__
          replacement: http
        - target_label: __address__
          replacement: blackbox-exporter:9115

      # 这里监控 blackbox exporter 自身
      - job_name: 'blackbox_exporter'
        static_configs:
          - targets:
            - blackbox-exporter:9115

注意上文的配置中存在 __meta_kubernetes_service_annotation_prometheus_io_http_probe，需要在业务 service 的 annotation 做对应的配置，prometheus 才会去检测，比如:

apiVersion: v1
kind: Service
metadata:
  name: grafana
  namespace: monitoring
  annotations:
    # 这里是关键，会被 prometheus 读取
    prometheus.io/http-probe: "true"
    prometheus.io/http-probe-path: "/"
spec:
  type: ClusterIP
  clusterIP: 10.43.87.148
  selector:
    name: grafana
  ports:
    - port: 3000
      targetPort: 3000

最后看效果: