Burrow failed to connect kafka cluster (kafka: client has run out of available brokers to talk to) #760
Description
Hi,
We have issue with burrow, which fails to connect kafka cluster, with below error.
Kafka.log
We see below info in server.log file.
[2022-07-26 15:01:24,823] INFO [SocketServer brokerId=1] Failed authentication with /xx.xx.xx.xx (SSL handshake failed) (org.apache.kafka.common.network.Selector)
Burrow.log
{"level":"info","ts":1657739261.2620974,"msg":"configuring","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739261.2879257,"msg":"configuring","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739261.287961,"msg":"configuring","type":"coordinator","name":"cluster"}
{"level":"info","ts":1657739261.2879865,"msg":"configuring","type":"module","coordinator":"cluster","class":"kafka","name":"kafka"}
{"level":"info","ts":1657739261.2911863,"msg":"configuring","type":"coordinator","name":"consumer"}
{"level":"info","ts":1657739261.2912247,"msg":"configuring","type":"module","coordinator":"consumer","class":"kafka","name":"burrow"}
{"level":"info","ts":1657739261.2921846,"msg":"starting","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3041592,"msg":"Connected to xx.xx.xx.xx:xxxx","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.321232,"msg":"Authenticated: id=144154910105734385, timeout=6000","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3213053,"msg":"Re-submitting 0 credentials after reconnect","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3518999,"msg":"starting","type":"coordinator","name":"storage"}
{"level":"info","ts":1657739261.351937,"msg":"starting","type":"module","coordinator":"storage","class":"inmemory","name":"default"}
{"level":"info","ts":1657739261.3519754,"msg":"starting","type":"coordinator","name":"evaluator"}
{"level":"info","ts":1657739261.3519819,"msg":"starting","type":"module","coordinator":"evaluator","class":"caching","name":"default"}
{"level":"info","ts":1657739261.3519928,"msg":"starting","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739261.352134,"msg":"started listener","type":"coordinator","name":"httpserver","listener":"[::]:9100"}
{"level":"info","ts":1657739261.3521514,"msg":"starting","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739261.3521652,"msg":"starting","type":"coordinator","name":"cluster"}
{"level":"info","ts":1657739261.3521693,"msg":"starting","type":"module","coordinator":"cluster","class":"kafka","name":"kafka"}
{"level":"info","ts":1657739261.4988637,"msg":"starting evaluations","type":"coordinator","name":"notifier"}
{"level":"error","ts":1657739263.573983,"msg":"failed to start client","type":"module","coordinator":"cluster","class":"kafka","name":"kafka","error":"kafka: client has run out of available brokers to talk to (Is your cluster reachable?)"}
{"level":"info","ts":1657739263.5740325,"msg":"stopping","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739263.5740402,"msg":"shutdown","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739263.5740795,"msg":"stopping","type":"coordinator","name":"evaluator"}
{"level":"info","ts":1657739263.5740893,"msg":"stopping","type":"module","coordinator":"evaluator","class":"caching","name":"default"}
{"level":"info","ts":1657739263.5740964,"msg":"stopping","type":"coordinator","name":"storage"}
{"level":"info","ts":1657739263.574107,"msg":"stopping","type":"module","coordinator":"storage","class":"inmemory","name":"default"}
{"level":"info","ts":1657739263.5741801,"msg":"stopping","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739263.5933099,"msg":"Recv loop terminated: err=EOF","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739263.5933375,"msg":"Send loop terminated: err=","type":"coordinator","name":"zookeeper"}
Burrow.toml
[general]
pidfile="burrow.pid"
stdout-logfile="burrow.out"
[logging]
filename="burrow.log"
level="info"
maxsize=100
maxbackups=5
maxage=7
use-localtime=true
use-compression=true
[zookeeper]
servers=[ "host:port","host:port","host:port" ]
timeout=6
root-path="/burrow/"
[client-profile.tlsonly]
client-id=""
kafka-version="0.10.2.1"
tls="tlsprofile"
[tls.tlsprofile]
certfile=""
keyfile=""
cafile=""
noverify=true
[cluster.kafka]
class-name="kafka"
servers=[ "host:port","host:port","host:port" ]
client-profile="tlsonly"
topic-refresh=120
offset-refresh=30
[consumer.burrow]
class-name="kafka"
cluster="kafka"
servers=[ "host:port","host:port","host:port" ]
client-profile="tlsonly"
group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
group-whitelist=""
[httpserver.burrowListener]
address=":9100"
tls="tlsprofile"
Any debugging help would be appriciated.
Regards,
Swapnil