docker-compose.yml

---
version: "3"
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.0
    hostname: zookeeper
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  broker:
    image: confluentinc/cp-kafka:7.3.0
    container_name: broker
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1

  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    restart: always
    ports:
      - 9870:9870
      - 9000:9000
    networks:
      - hadoop
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
      - HDFS_CONF_dfs_replication=1
    env_file:
      - ./hadoop.env

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    restart: always
    ports:
      - 9864:9864
    networks:
      - hadoop
    hostname: datanode
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env

  resourcemanager:
    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
    container_name: resourcemanager
    restart: always
    ports:
      - 8088:8088
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  nodemanager1:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
    container_name: nodemanager
    restart: always
    ports:
      - 8042:8042
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    container_name: historyserver
    restart: always
    ports:
      - 8188:8188
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  # spark-master:
  #   image: bde2020/spark-master:2.1.0-hadoop2.8-hive-java8
  #   container_name: spark-master
  #   ports:
  #     - 8080:8080
  #     - 7077:7077
  #   environment:
  #     - INIT_DAEMON_STEP=setup_spark
  #     - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
  #   networks:
  #     - hadoop

  # spark-worker:
  #   image: bde2020/spark-worker:2.1.0-hadoop2.8-hive-java8
  #   container_name: spark-worker-1
  #   depends_on:
  #     - spark-master
  #   ports:
  #     - 8081:8081
  #   environment:
  #     - "SPARK_MASTER=spark://spark-master:7077"
  #     - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
  #   networks:
  #     - hadoop

  spark-notebook:
    build: ./spark-notebooks
    container_name: spark-notebook
    ports:
      - 8888:8888
      - 4040:4040
    networks:
      - hadoop
    volumes:
      - ./spark-notebooks/workspace:/workspace

networks:
  hadoop:
volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver: