Skip to content
This repository has been archived by the owner on May 16, 2023. It is now read-only.
This repository has been archived by the owner on May 16, 2023. It is now read-only.

Filebeat to kafka output does not work
#694

Closed
#699
Closed
Filebeat to kafka output does not work#694
#699

Description

@shivoham77

shivoham77
openedon Jun 26, 2020

Chart version: 6.8.11SNAPSHOT

Kubernetes version: client - 1.17 server- 1.15

Kubernetes provider: AWS EKS E.g. GKE (Google Kubernetes Engine)

Helm Version: 2.16.7

**helm get release output

REVISION: 1
RELEASED: Fri Jun 26 06:13:08 2020
CHART: filebeat-7.7.1
USER-SUPPLIED VALUES:
affinity: {}
envFrom: []
extraContainers: ""
extraEnvs:
- name: KAFKA_BROKERS
  value: b-1.test.cb80rx.c3.kafka.eu-central-1.amazonaws.com:9092
- name: TOPIC_APPLICATION_LOGS
  value: Alpication_Logs
extraInitContainers: []
extraVolumeMounts: []
extraVolumes: []
filebeatConfig:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      paths:
        - /var/log/containers/*.log
      processors:
      - add_kubernetes_metadata:
          host: ${NODE_NAME}
          matchers:
          - logs_path:
              logs_path: "/var/log/containers/"
    output.kafka:
       enabled: true
       hosts: ["b-1.logs-kafka-1-1-1-st.ycz3n5.c3.kafka.eu-central-1.amazonaws.com:9092","b-2.logs-kafka-1-1-1-st.ycz3n5.c3.kafka.eu-central-1.amazonaws.com:9092"]
       topic: "Application_logs"
       codec.format:
          string: '%{[@timestamp]} %{[message]}'

    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
    filebeat.modules:
    - module: nginx
    - module: kafka
fullnameOverride: ""
hostNetworking: false
hostPathRoot: /var/lib
image: docker.elastic.co/beats/filebeat
imagePullPolicy: IfNotPresent
imagePullSecrets: []
imageTag: 6.8.11-SNAPSHOT
labels: {}
livenessProbe:
  exec:
    command:
    - sh
    - -c
    - |
      #!/usr/bin/env bash -e
      curl --fail 127.0.0.1:5066
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 5
managedServiceAccount: true
nameOverride: ""
nodeSelector: {}
podAnnotations: {}
podSecurityContext:
  privileged: false
  runAsUser: 0
priorityClassName: ""
resources:
  limits:
    cpu: 1000m
    memory: 200Mi
  requests:
    cpu: 100m
    memory: 100Mi
secretMounts: []
serviceAccount: ""
terminationGracePeriod: 30
tolerations: []
updateStrategy: RollingUpdate

COMPUTED VALUES:
affinity: {}
envFrom: []
extraContainers: ""
extraEnvs:
- name: KAFKA_BROKERS
  value: b-1.test.cb80rx.c3.kafka.eu-central-1.amazonaws.com:9092
- name: TOPIC_APPLICATION_LOGS
  value: Alpication_Logs
extraInitContainers: []
extraVolumeMounts: []
extraVolumes: []
filebeatConfig:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      paths:
        - /var/log/containers/*.log
      processors:
      - add_kubernetes_metadata:
          host: ${NODE_NAME}
          matchers:
          - logs_path:
              logs_path: "/var/log/containers/"
    output.kafka:
       enabled: true
       hosts: ["b-1.logs-kafka-1-1-1-st.ycz3n5.c3.kafka.eu-central-1.amazonaws.com:9092","b-2.logs-kafka-1-1-1-st.ycz3n5.c3.kafka.eu-central-1.amazonaws.com:9092"]
       topic: "Application_logs"
       codec.format:
          string: '%{[@timestamp]} %{[message]}'

    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
    filebeat.modules:
    - module: nginx
    - module: kafka
fullnameOverride: ""
hostNetworking: false
hostPathRoot: /var/lib
image: docker.elastic.co/beats/filebeat
imagePullPolicy: IfNotPresent
imagePullSecrets: []
imageTag: 6.8.11-SNAPSHOT
labels: {}
livenessProbe:
  exec:
    command:
    - sh
    - -c
    - |
      #!/usr/bin/env bash -e
      curl --fail 127.0.0.1:5066
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 5
managedServiceAccount: true
nameOverride: ""
nodeSelector: {}
podAnnotations: {}
podSecurityContext:
  privileged: false
  runAsUser: 0
priorityClassName: ""
readinessProbe:
  exec:
    command:
    - sh
    - -c
    - |
      #!/usr/bin/env bash -e
      filebeat test output
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 5
resources:
  limits:
    cpu: 1000m
    memory: 200Mi
  requests:
    cpu: 100m
    memory: 100Mi
secretMounts: []
serviceAccount: ""
terminationGracePeriod: 30
tolerations: []
updateStrategy: RollingUpdate

HOOKS:
MANIFEST:

---
# Source: filebeat/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: fb-logs-filebeat-config
  labels:
    app: "fb-logs-filebeat"
    chart: "filebeat-7.7.1"
    heritage: "Tiller"
    release: "fb-logs"
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      paths:
        - /var/log/containers/*.log
      processors:
      - add_kubernetes_metadata:
          host: ${NODE_NAME}
          matchers:
          - logs_path:
              logs_path: "/var/log/containers/"
    output.kafka:
       enabled: true
       hosts: ["host1:9092","host2:9092"]
       topic: "Application_logs"
       codec.format:
          string: '%{[@timestamp]} %{[message]}'

    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
    filebeat.modules:
    - module: nginx
    - module: kafka
---
# Source: filebeat/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fb-logs-filebeat
  labels:
    app: "fb-logs-filebeat"
    chart: "filebeat-7.7.1"
    heritage: "Tiller"
    release: "fb-logs"
---
# Source: filebeat/templates/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: fb-logs-filebeat-cluster-role
  labels:
    app: "fb-logs-filebeat"
    chart: "filebeat-7.7.1"
    heritage: "Tiller"
    release: "fb-logs"
rules:
- apiGroups:
  - ""
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - list
  - watch
---
# Source: filebeat/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: fb-logs-filebeat-cluster-role-binding
  labels:
    app: "fb-logs-filebeat"
    chart: "filebeat-7.7.1"
    heritage: "Tiller"
    release: "fb-logs"
roleRef:
  kind: ClusterRole
  name: fb-logs-filebeat-cluster-role
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fb-logs-filebeat
  namespace: filebeat
---
# Source: filebeat/templates/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fb-logs-filebeat
  labels:
    app: "fb-logs-filebeat"
    chart: "filebeat-7.7.1"
    heritage: "Tiller"
    release: "fb-logs"
spec:
  selector:
    matchLabels:
      app: "fb-logs-filebeat"
      release: "fb-logs"
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      annotations:

        configChecksum: 27e139df1099c2e0e943409748e73720838794328036f5724b1369a43e905f7
      name: "fb-logs-filebeat"
      labels:
        app: "fb-logs-filebeat"
        chart: "filebeat-7.7.1"
        heritage: "Tiller"
        release: "fb-logs"
    spec:
      serviceAccountName: fb-logs-filebeat
      terminationGracePeriodSeconds: 30
      volumes:
      - name: filebeat-config
        configMap:
          defaultMode: 0600
          name: fb-logs-filebeat-config
      - name: data
        hostPath:
          path: /var/lib/fb-logs-filebeat-filebeat-data
          type: DirectoryOrCreate
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: varlog
        hostPath:
          path: /var/log
      - name: varrundockersock
        hostPath:
          path: /var/run/docker.sock
      containers:
      - name: "filebeat"
        image: "docker.elastic.co/beats/filebeat:6.8.11-SNAPSHOT"
        imagePullPolicy: "IfNotPresent"
        args:
        - "-e"
        - "-E"
        - "http.enabled=true"
        livenessProbe:
          exec:
            command:
            - sh
            - -c
            - |
              #!/usr/bin/env bash -e
              curl --fail 127.0.0.1:5066
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 5

        readinessProbe:
          exec:
            command:
            - sh
            - -c
            - |
              #!/usr/bin/env bash -e
              filebeat test output
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 5

        resources:
          limits:
            cpu: 1000m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi

        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: KAFKA_BROKERS
          value: host1:9092
        - name: TOPIC_APPLICATION_LOGS
          value: Alpication_Logs

        securityContext:
          privileged: false
          runAsUser: 0

        volumeMounts:
        - name: filebeat-config
          mountPath: /usr/share/filebeat/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: varlog
          mountPath: /var/log
          readOnly: true
        # Necessary when using autodiscovery; avoid mounting it otherwise
        # See: https://www.elastic.co/guide/en/beats/filebeat/7.7/configuration-autodiscover.html
        - name: varrundockersock
          mountPath: /var/run/docker.sock
          readOnly: true

**

e.g. `helm get output fb-logs` (replace `elasticsearch` with the name of your helm release)

*Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting.*

*If you find some secrets in plain text in `helm get release` output you should use [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) to managed them is a secure way (see [Security Example](https://github.com/elastic/helm-charts/blob/master/elasticsearch/examples/security/security.yml#L23-L38)).*

<details>
<summary>Output of helm get release</summary>

Describe the bug:

When installed, The pods do not run, not topic with logs sent to kafka. Kafka Modules not enabled.
in the pod "filebeat module list" kafka not present in "enabled".
Readiness probe fail: kafka output doesn't support testing.

Steps to reproduce:

  1. Just install 6.8.11-SNAPSHOT with output Kafka configuration.
  2. Configuration below
 4 filebeatConfig:
     filebeat.yml: |
       filebeat.inputs:
       - type: log
         paths:
           - /var/log/containers/*.log
         processors:
         - add_kubernetes_metadata:
            host: ${NODE_NAME}
            matchers:
            - logs_path:
                logs_path: "/var/log/containers/"
       output.kafka:
         enabled: true
         hosts: ["host1:9092","host2:9092"]
         topic: "Application_logs"
         codec.format:
            string: '%{[@timestamp]} %{[message]}'
 
       filebeat.config.modules:
         path: ${path.config}/modules.d/*.yml
         filebeat.modules:
         - module: nginx
         - module: kafka
  1. Delete readiness probe from daemonset than pods and containers run. kubectl exec into pod and test modules enabled or not

**Expected behavior: Modules enabled, readiness probe sould not fail, filebeat should be sending topic and logs to kafka, **

Provide logs and/or server output (if relevant):

Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting

2020-06-26T04:44:58.954Z	INFO	instance/beat.go:611	Home path: [/usr/share/filebeat] Config path: [/usr/share/filebeat] Data path: [/usr/share/filebeat/data] Logs path: [/usr/share/filebeat/logs]
2020-06-26T04:44:58.954Z	INFO	instance/beat.go:618	Beat UUID: 0176e57b-5e43-4d2c-b4ad-5800fb60ae4a
2020-06-26T04:44:58.954Z	INFO	[seccomp]	seccomp/seccomp.go:116	Syscall filter successfully installed
2020-06-26T04:44:58.954Z	INFO	[beat]	instance/beat.go:931	Beat info	{"system_info": {"beat": {"path": {"config": "/usr/share/filebeat", "data": "/usr/share/filebeat/data", "home": "/usr/share/filebeat", "logs": "/usr/share/filebeat/logs"}, "type": "filebeat", "uuid": "0176e57b-5e43-4d2c-b4ad-5800fb60ae4a"}}}
2020-06-26T04:44:58.954Z	INFO	[beat]	instance/beat.go:940	Build info	{"system_info": {"build": {"commit": "970a26b80eb078a76ba0fd4bc1b9328181665080", "libbeat": "6.8.11", "time": "2020-06-25T13:52:01.000Z", "version": "6.8.11"}}}
2020-06-26T04:44:58.954Z	INFO	[beat]	instance/beat.go:943	Go runtime info	{"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.10.8"}}}
2020-06-26T04:44:58.955Z	INFO	[beat]	instance/beat.go:947	Host info	{"system_info": {"host": {"architecture":"x86_64","boot_time":"2020-06-02T15:05:16Z","containerized":false,"name":"fb-logs-filebeat-jrzmw","ip":["127.0.0.1/8","10.8.136.78/32"],"kernel_version":"4.14.177-139.253.amzn2.x86_64","mac":["5a:d6:4d:d8:dd:f0"],"os":{"family":"redhat","platform":"centos","name":"CentOS Linux","version":"7 (Core)","major":7,"minor":8,"patch":2003,"codename":"Core"},"timezone":"UTC","timezone_offset_sec":0}}}
2020-06-26T04:44:58.955Z	INFO	[beat]	instance/beat.go:976	Process info	{"system_info": {"process": {"capabilities": {"inheritable":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"permitted":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"effective":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"bounding":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"ambient":null}, "cwd": "/usr/share/filebeat", "exe": "/usr/share/filebeat/filebeat", "name": "filebeat", "pid": 1, "ppid": 0, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2020-06-26T04:44:57.970Z"}}}
2020-06-26T04:44:58.955Z	INFO	instance/beat.go:280	Setup Beat: filebeat; Version: 6.8.11
2020-06-26T04:44:58.956Z	INFO	[publisher]	pipeline/module.go:110	Beat name: fb-logs-filebeat-jrzmw
2020-06-26T04:44:58.957Z	INFO	beater/filebeat.go:101	Enabled modules/filesets: nginx (access, error), kafka (log),  ()
2020-06-26T04:44:58.958Z	INFO	instance/beat.go:402	filebeat start running.
2020-06-26T04:44:58.958Z	WARN	[cfgwarn]	api/server.go:34	EXPERIMENTAL: Metrics endpoint is enabled.
2020-06-26T04:44:58.958Z	INFO	api/server.go:38	Starting stats endpoint
2020-06-26T04:44:58.958Z	WARN	beater/filebeat.go:180	Filebeat is unable to load the Ingest Node pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the Ingest Node pipelines or are using Logstash pipelines, you can ignore this warning.
2020-06-26T04:44:58.958Z	INFO	[monitoring]	log/log.go:117	Starting metrics logging every 30s
2020-06-26T04:44:58.958Z	INFO	registrar/registrar.go:134	Loading registrar data from /usr/share/filebeat/data/registry
2020-06-26T04:44:58.958Z	INFO	registrar/registrar.go:141	States Loaded from registrar: 0
2020-06-26T04:44:58.958Z	WARN	beater/filebeat.go:367	Filebeat is unable to load the Ingest Node pipelines for the configured modules because the Elasticsearch output is not configured/enabled. If you have already loaded the Ingest Node pipelines or are using Logstash pipelines, you can ignore this warning.
2020-06-26T04:44:58.958Z	INFO	api/server.go:49	Metrics endpoint listening on: localhost:5066
2020-06-26T04:44:58.958Z	INFO	crawler/crawler.go:72	Loading Inputs: 4
2020-06-26T04:44:58.959Z	INFO	kubernetes/util.go:71	kubernetes: Using node ip-10-8-129-106.eu-central-1.compute.internal provided in the config
2020-06-26T04:44:58.959Z	INFO	kubernetes/watcher.go:182	kubernetes: Performing a resource sync for *v1.PodList
2020-06-26T04:44:58.966Z	INFO	kubernetes/watcher.go:198	kubernetes: Resource sync done
2020-06-26T04:44:58.966Z	INFO	kubernetes/watcher.go:242	kubernetes: Watching API for resource events
2020-06-26T04:44:58.966Z	INFO	log/input.go:148	Configured paths: [/var/log/containers/*.log]
2020-06-26T04:44:58.966Z	INFO	input/input.go:114	Starting input of type: log; ID: 9205147391741484860
2020-06-26T04:44:58.966Z	INFO	log/input.go:148	Configured paths: [/var/log/nginx/access.log*]
2020-06-26T04:44:58.966Z	INFO	input/input.go:114	Starting input of type: log; ID: 14420054754517151854
2020-06-26T04:44:58.967Z	INFO	log/input.go:148	Configured paths: [/var/log/nginx/error.log*]
2020-06-26T04:44:58.967Z	INFO	input/input.go:114	Starting input of type: log; ID: 1147347347273245827
2020-06-26T04:44:58.967Z	INFO	log/input.go:148	Configured paths: [/opt/kafka*/logs/controller.log* /opt/kafka*/logs/server.log* /opt/kafka*/logs/state-change.log* /opt/kafka*/logs/kafka-*.log*]
2020-06-26T04:44:58.967Z	INFO	input/input.go:114	Starting input of type: log; ID: 133507599549825125
2020-06-26T04:44:58.967Z	INFO	crawler/crawler.go:106	Loading and starting Inputs completed. Enabled inputs: 4
2020-06-26T04:44:58.967Z	INFO	cfgfile/reload.go:150	Config reloader started
2020-06-26T04:44:58.968Z	INFO	cfgfile/reload.go:205	Loading of config files completed.
2020-06-26T04:45:28.960Z	INFO	[monitoring]	log/log.go:144	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":20,"time":{"ms":21}},"total":{"ticks":50,"time":{"ms":52},"value":50},"user":{"ticks":30,"time":{"ms":31}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":7},"info":{"ephemeral_id":"0c81ceb5-9ccb-4c98-bb11-633da4181944","uptime":{"ms":30019}},"memstats":{"gc_next":4194304,"memory_alloc":2146000,"memory_total":5660848,"rss":27348992}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0},"reloads":1},"output":{"type":"kafka"},"pipeline":{"clients":4,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"cpu":{"cores":2},"load":{"1":0.04,"15":0.16,"5":0.27,"norm":{"1":0.02,"15":0.08,"5":0.135}}}}}}
2020-06-26T04:45:58.960Z	INFO	[monitoring]	log/log.go:144	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":20},"total":{"ticks":50,"time":{"ms":6},"value":50},"user":{"ticks":30,"time":{"ms":6}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":7},"info":{"ephemeral_id":"0c81ceb5-9ccb-4c98-bb11-633da4181944","uptime":{"ms":60019}},"memstats":{"gc_next":4194304,"memory_alloc":2580616,"memory_total":6095464}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":4,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":0.02,"15":0.15,"5":0.25,"norm":{"1":0.01,"15":0.075,"5":0.125}}}}}}
2020-06-26T04:46:28.960Z	INFO	[monitoring]	log/log.go:144	Non-zero metrics in the last 30s	{"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":20},"total":{"ticks":60,"time":{"ms":6},"value":60},"user":{"ticks":40,"time":{"ms":6}}},"handles":{"limit":{"hard":65536,"soft":65536},"open":7},"info":{"ephemeral_id":"0c81ceb5-9ccb-4c98-bb11-633da4181944","uptime":{"ms":90019}},"memstats":{"gc_next":4194304,"memory_alloc":3004024,"memory_total":6518872}},"filebeat":{"harvester":{"open_files":0,"running":0}},"libbeat":{"config":{"module":{"running":0}},"pipeline":{"clients":4,"events":{"active":0}}},"registrar":{"states":{"current":0}},"system":{"load":{"1":0.17,"15":0.16,"5":0.25,"norm":{"1":0.085,"15":0.08,"5":0.125}}}}}}

Any additional context:

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Assignees

No one assigned

    Labels

    bugSomething isn't workingSomething isn't workingfilebeat

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions