javax.net.ssl.SSLHandshakeException : Kafka Composer

[CodeWithAdarsha]

I was trying to set up kafka cluster with 2 broker and 2 zookeeper with SSL enabled and Zookeeper working fine with SSL but its failing at running broker with below error. Looks like certificate path from broker (inside composer) not picking. I can see certificate placed inside docker successfully.

Can you please help me with this ?

broker-2 | [2023-01-02 05:17:53,466] ERROR [KafkaServer id=2] Fatal error during KafkaServer startup. Prepare to shutdown (kafka.server.KafkaServer)
broker-2 | org.apache.kafka.common.config.ConfigException: Invalid value javax.net.ssl.SSLHandshakeException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target for configuration A client SSLEngine created with the provided settings can't connect to a server SSLEngine created with those settings.
broker-2 | at org.apache.kafka.common.security.ssl.SslFactory.configure(SslFactory.java:102)
broker-2 | at org.apache.kafka.common.network.SslChannelBuilder.configure(SslChannelBuilder.java:73)
broker-2 | at org.apache.kafka.common.network.ChannelBuilders.create(ChannelBuilders.java:192)
broker-2 | at org.apache.kafka.common.network.ChannelBuilders.serverChannelBuilder(ChannelBuilders.java:107)
broker-2 | at kafka.network.Processor.(SocketServer.scala:853)
broker-2 | at kafka.network.SocketServer.newProcessor(SocketServer.scala:442)
broker-2 | at kafka.network.SocketServer.$anonfun$addDataPlaneProcessors$1(SocketServer.scala:299)
broker-2 | at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:190)
broker-2 | at kafka.network.SocketServer.addDataPlaneProcessors(SocketServer.scala:297)
broker-2 | at kafka.network.SocketServer.$anonfun$createDataPlaneAcceptorsAndProcessors$1(SocketServer.scala:262)
broker-2 | at kafka.network.SocketServer.$anonfun$createDataPlaneAcceptorsAndProcessors$1$adapted(SocketServer.scala:259)
broker-2 | at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:563)
broker-2 | at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:561)
broker-2 | at scala.collection.AbstractIterable.foreach(Iterable.scala:919)
broker-2 | at kafka.network.SocketServer.createDataPlaneAcceptorsAndProcessors(SocketServer.scala:259)
broker-2 | at kafka.network.SocketServer.startup(SocketServer.scala:131)
broker-2 | at kafka.server.KafkaServer.startup(KafkaServer.scala:285)
broker-2 | at kafka.Kafka$.main(Kafka.scala:109)
broker-2 | at kafka.Kafka.main(Kafka.scala)
broker-2 | [2023-01-02 05:17:53,479] INFO [KafkaServer id=2] shutting down (kafka.server.KafkaServer)
broker-2 | [2023-01-02 05:17:53,482] INFO [SocketServer listenerType=ZK_BROKER, nodeId=2] Stopping socket server request processors (kafka.network.SocketServer)
broker-2 | [2023-01-02 05:17:53,495] INFO [SocketServer listenerType=ZK_BROKER, nodeId=2] Stopped socket server request processors (kafka.network.SocketServer)

Docker Image ::

version: '3'
services:
  znode-one:
    image: zookeeper:latest
    container_name: zNode-1
    restart: unless-stopped
    ports:
      - "2181:2181"    #Client
      - "2888:2888"    #Leader
      - "3888:3888"    #Election
      - "10020:10020"  #JMX
      - "10021:10021"
    volumes:
      - ./cert/keystore/broker-1.keystore.jks:/security/broker-1.keystore.jks
      - ./cert/truststore/broker-1.truststore.jks:/security/broker-1.truststore.jks
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVERS: server.1=znode-one:2888:3888;2181 server.2=znode-two:2888:3888;2181
      ZOO_CFG_EXTRA: "sslQuorum=false
                      portUnification=true
                      serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory

                      ssl.quorum.hostnameVerification=false
                      ssl.quorum.keyStore.location=/security/broker-1.keystore.jks
                      ssl.quorum.keyStore.password=<password>
                      ssl.quorum.trustStore.location=/security/broker-1.truststore.jks
                      ssl.quorum.trustStore.password=<password>

                      secureClientPort=2281
                      ssl.hostnameVerification=false
                      ssl.keyStore.location=/security/broker-1.keystore.jks
                      ssl.keyStore.password=<password>
                      ssl.trustStore.location=/security/broker-1.truststore.jks
                      ssl.trustStore.password=<password>"
    networks:
      laso-dev:

  znode-two:
    image: zookeeper:latest
    container_name: zNode-2
    restart: unless-stopped
    ports:
      - "2182:2182"
      - "2889:2889"
      - "3889:3889"
      - "10022:10022"  # JMX
      - "10023:10023"
    volumes:
      - ./cert/keystore/broker-1.keystore.jks:/security/broker-1.keystore.jks
      - ./cert/truststore/broker-1.truststore.jks:/security/broker-1.truststore.jks
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_CLIENT_PORT: 2182
      ZOOKEEPER_SERVERS: server.1=znode-one:2888:3888;2181 server.2=znode-two:2888:3888;2181
      ZOO_CFG_EXTRA: "sslQuorum=false
                     portUnification=true
                     serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory

                     ssl.quorum.hostnameVerification=false
                     ssl.quorum.keyStore.location=/security/broker-1.keystore.jks
                     ssl.quorum.keyStore.password=<password>
                     ssl.quorum.trustStore.location=/security/broker-1.truststore.jks
                     ssl.quorum.trustStore.password=<password>

                     secureClientPort=2281
                     ssl.hostnameVerification=false
                     ssl.keyStore.location=/security/broker-1.keystore.jks
                     ssl.keyStore.password=<password>
                     ssl.trustStore.location=/security/broker-1.truststore.jks
                     ssl.trustStore.password=<password>"
    networks:
      laso-dev:

  kafka1:
    image: wurstmeister/kafka:latest
    restart: "on-failure"
    container_name: broker-1
    hostname: kafka1
    depends_on:
      - znode-one
      - znode-two
    ports:
      - "9092:9092"
      - "9192:9192"
      - "10030:10030"
      - "10031:10031"
    volumes:
      - ./cert/keystore/broker-1.keystore.jks:/certs/broker-1.keystore.jks
      - ./cert/truststore/broker-1.truststore.jks:/certs/broker-1.truststore.jks
    environment:
      # KAFKA_LOG_DIRS: /kafka/logs
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_HOST_NAME: kafka1
      KAFKA_ZOOKEEPER_CONNECT: znode-one:2181,znode-two:2182
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SSL:SSL 
      KAFKA_ADVERTISED_LISTENERS: SSL
      KAFKA_LISTENERS: SSL
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: SSL
      # KAFKA_INTER_BROKER_LISTENER_NAME: SSL
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
      KAFKA_BROKER_RACK: "r1"
      # ZOOKEEPER SSL Enable
      KAFKA_ZOOKEEPER_SSL_CLIENT_ENABLE: "true"
      KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET: org.apache.zookeeper.ClientCnxnSocketNetty
      KAFKA_ZOOKEEPER_SSL_KEYSTORE_LOCATION: /certs/broker-1.keystore.jks
      KAFKA_ZOOKEEPER_SSL_KEYSTORE_PASSWORD: <password>
      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /certs/broker-1.truststore.jks
      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: <password>
      # KAFKA KEEPER SSL Enable
      KAFKA_SSL_CLIENT_AUTH: none
      KAFKA_SSL_KEY_PASSWORD: <password>  
      KAFKA_SSL_KEYSTORE_LOCATION: /certs/broker-1.keystore.jks
      KAFKA_SSL_KEYSTORE_PASSWORD: <password>
      KAFKA_SSL_TRUSTSTORE_LOCATION: /certs/broker-1.truststore.jks
      KAFKA_SSL_TRUSTSTORE_PASSWORD: <password>
      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ''
      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_ZOOKEEPER_SESSION_TIMEOUT: "6000"
      KAFKA_RESTART_ATTEMPTS: "10"
      KAFKA_RESTART_DELAY: "5"
      ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0"
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_CREATE_TOPICS: "test:1:1"
      KAFKA_AUTO_LEADER_REBALANCE_ENABLE: 'true'
      KAFKA_NUM_PARTITIONS: 20
      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 15
      KAFKA_DELETE_TOPIC_ENABLE: "true"
      KAFKA_LOG_RETENTION_HOURS: 3
      KAFKA_LOG_ROLL_HOURS: 1
    networks:
      laso-dev:

  kafka2:
    image: wurstmeister/kafka:latest
    container_name: broker-2
    restart: "on-failure"
    hostname: kafka2
    depends_on:
      - znode-one
      - znode-two
    ports:
      - "9094:9094"
      - "9194:9194"
      - "10032:10032"
      - "10033:10033"
    volumes:
      - ./cert/keystore/broker-1.keystore.jks:/certs/broker-1.keystore.jks
      - ./cert/truststore/broker-1.truststore.jks:/certs/broker-1.truststore.jks
    environment:
       # KAFKA_LOG_DIRS: /kafka/logs
      KAFKA_BROKER_ID: 2
      KAFKA_ADVERTISED_HOST_NAME: kafka2
      KAFKA_ZOOKEEPER_CONNECT: znode-one:2181,znode-two:2182
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SSL:SSL # PLAINTEXT:PLAINTEXT,# INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: SSL://kafka2:9194 
      KAFKA_LISTENERS: SSL://kafka2:9194 
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: SSL
      # KAFKA_INTER_BROKER_LISTENER_NAME: SSL
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
      KAFKA_BROKER_RACK: "r2"
      KAFKA_ZOOKEEPER_SESSION_TIMEOUT: "6000" 
      KAFKA_RESTART_ATTEMPTS: "10"
      KAFKA_RESTART_DELAY: "5"
      ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0"
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_AUTO_LEADER_REBALANCE_ENABLE: 'true'
      KAFKA_NUM_PARTITIONS: 20
      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 15  
      KAFKA_DELETE_TOPIC_ENABLE: "true"
      KAFKA_LOG_RETENTION_HOURS: 3
      KAFKA_LOG_ROLL_HOURS: 1

      # ZOO KEEPER SSL Enable
      KAFKA_ZOOKEEPER_SSL_CLIENT_ENABLE: "true"
      KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET: org.apache.zookeeper.ClientCnxnSocketNetty
      KAFKA_ZOOKEEPER_SSL_KEYSTORE_LOCATION: /certs/broker-1.keystore.jks
      KAFKA_ZOOKEEPER_SSL_KEYSTORE_PASSWORD: <password>
      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /certs/broker-1.truststore.jks
      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: <password>
      # KAFKA KEEPER SSL Enable
      KAFKA_SSL_CLIENT_AUTH: none
      KAFKA_SSL_KEY_PASSWORD: <password>
      KAFKA_SSL_KEYSTORE_LOCATION: /certs/broker-1.keystore.jks
      KAFKA_SSL_KEYSTORE_PASSWORD: <password>
      KAFKA_SSL_TRUSTSTORE_LOCATION: /certs/broker-1.truststore.jks
      KAFKA_SSL_TRUSTSTORE_PASSWORD: <password>
      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ''
    networks:
      laso-dev:


networks:
  laso-dev:
    driver: bridge

---