kafka-docker-compose.yaml

version: "3"

services:
# Setting Up HDFS & YARN #
  namenode:
    image: mrugankray/namenode-spark-airflow-flume-zepplin:1.1
    container_name: namenode
    restart: always
    ports:
      - 9870:9870
      - 9000:9000
      - 8082:8082 # zeppelin ui
      - 8080:8080 # spark master web ui
      - 8081:8081 # spark slave web ui
      - 4040:4040 # spark driver web ui
      - 3000:3000 # airflow ui
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
      - hadoop_namenode_conda:/root/anaconda
      - hadoop_namenode_spark:/opt/spark
      - hadoop_namenode_zeppelin:/opt/zeppelin
      - ./configs/zeppelin-site.xml:/opt/zeppelin/conf/zeppelin-site.xml
      - ./configs/zeppelin-env.sh:/opt/zeppelin/conf/zeppelin-env.sh
      - ./configs/namenode_bashrc.txt:/root/.bashrc
      - ./configs/namenode_airflow.cfg:/root/airflow/airflow.cfg
      - ./dags:/root/airflow/dags
      - airflow_namenode:/root/airflow
      - ./configs/namenode/flume/flume-env.sh:/opt/flume/conf/flume-env.sh
      - ./flume_config/flume.conf:/opt/flume/conf/flume.conf
      - hadoop_namenode_flume:/opt/flume
    environment:
      - CLUSTER_NAME=hadoop-learning
      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
    env_file:
      - ./hadoop.env

  datanode:
    image: mrugankray/datanode-python:1.0
    container_name: datanode
    restart: always
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
      - hadoop_datanode_conda:/root/anaconda
      - ./configs/datanode_bashrc.txt:/root/.bashrc
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
      CORE_CONF_fs_defaultFS: "hdfs://namenode:9000"
    ports:
      - "9864:9864"
    env_file:
      - ./hadoop.env

  resourcemanager:
    image: mrugankray/resourcemanager-python:1.0
    container_name: resourcemanager
    restart: always
    volumes:
      - hadoop_resourcemanager_conda:/root/anaconda
      - ./configs/resourcemanager_bashrc.txt:/root/.bashrc
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
    ports:
      - "8088:8088"
    env_file:
      - ./hadoop.env

  nodemanager:
    image: mrugankray/nodemanager-python:1.0
    container_name: nodemanager
    restart: always
    volumes:
      - hadoop_nodemanager_conda:/root/anaconda
      - ./configs/nodemanager_bashrc.txt:/root/.bashrc
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    ports:
      - "8042:8042"
      - "19888:19888" # to access job history
    env_file:
      - ./hadoop.env

  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    container_name: historyserver
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    ports:
      - "8188:8188"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./hadoop.env
  
# Setting Kafka cluster #
  zookeeper:
    image: confluentinc/cp-zookeeper:5.4.0
    hostname: zookeeper
    container_name: zookeeper
    depends_on:
      - namenode
      - datanode
      - resourcemanager
      - nodemanager
      - historyserver
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_log:/var/lib/zookeeper/log

  kafka-broker:
    image: confluentinc/cp-server:5.4.0
    hostname: kafka-broker
    container_name: kafka-broker
    depends_on:
      - zookeeper
    ports:
      - "29092:29092"
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:29092
      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: "true"
      CONFLUENT_SUPPORT_CUSTOMER_ID: "anonymous"
    volumes:
      - kafka_broker:/var/lib/kafka/data

  schema-registry:
    image: confluentinc/cp-schema-registry:5.4.0
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      - zookeeper
      - kafka-broker
    deploy:
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
        window: 120s
    ports:
      - "8083:8083"
    environment:
      # SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: http://kafka-broker:29092
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      # setting SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL to zookeper. schema registry will get kafka server url from zookeper
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181"
      SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8083
      SCHEMA_REGISTRY_KAFKASTORE_ZK_SESSION_TIMEOUT_MS: 300000
      SCHEMA_REGISTRY_KAFKASTORE_INIT_TIMEOUT_MS: 300000

volumes:
  hadoop_namenode:
  hadoop_namenode_conda:
  hadoop_namenode_spark:
  hadoop_namenode_zeppelin:
  hadoop_namenode_flume:
  hadoop_datanode:
  hadoop_datanode_conda:
  hadoop_resourcemanager_conda:
  hadoop_nodemanager_conda:
  hadoop_historyserver:
  airflow_namenode:
  zookeeper_data:
  zookeeper_log:
  kafka_broker: