docker-compose.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

x-airflow-common:
  &airflow-common
  image: $GITHUB_CONTAINER_REPO
  environment:
    &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: $DB_CONNECTION_STRING
    AIRFLOW__CELERY__RESULT_BACKEND: DB_CONNECTION_STRING
    AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0
    AIRFLOW__CORE__FERNET_KEY: ''
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'true'
    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
    AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'false'
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
  volumes:
    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags
    - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
    - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config
    - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
  user: "${AIRFLOW_UID:-50000}:0"
  # depends_on:
  #   &airflow-common-depends-on

version: "3.9"
services:

  producer:
    image: $GITHUB_CONTAINER_REPO
    container_name: ${PROJECT_NAME}-producer
    hostname: producer
    restart: always
    stop_grace_period: 10s
    environment:
      KAFKA_BOOTSTRAP_HOST: kafka:9092
    volumes:
      - ./conf:/workspace/conf
      - ./data:/workspace/data:r
      - ./aks:/workspace/aks:rwx
    command: python -m aks.send_stream
    depends_on:
      - kafka

  mlflow:
    image: $GITHUB_CONTAINER_REPO
    container_name: ${PROJECT_NAME}-mlflow
    hostname: mlflow
    restart: always
    stop_grace_period: 10s
    build: 
      context: .
      args:
        PYTHON_TAG: $PYTHON_TAG
    environment:
      MLFLOW_BACKEND_STORE_URI: $DB_CONNECTION_STRING
      MLFLOW_S3_ENDPOINT_URL: http://minio:9000
      AWS_ACCESS_KEY_ID: $DB_USER
      AWS_SECRET_ACCESS_KEY: $DB_PASSWORD
      BENTOML_HOME: $BENTOML_HOME 
      BENTOML_DO_NOT_TRACK: "True"
      BENTOML_CONFIG: $BENTOML_CONFIG
      BENTOML_BUCKET: s3://bento
      PYSPARK_SUBMIT_ARGS: $PYSPARK_SUBMIT_ARGS
      SPARK_MASTER_URL: spark://spark:7077
      KAFKA_BOOTSTRAP_HOST: kafka:9092
    ports:
      - 5000:5000
      - 3001:3000
    volumes:
      - mlflow-prometheus-data:/prometheus
      - ./bentoml_configuration.yaml:${BENTOML_CONFIG}:rw
      - ./bento:${BENTOML_HOME}
      - ./conf:/workspace/conf
      - ./data:/workspace/data:r
      - ./models:/workspace/models:rw
      - ./outdir:/workspace/outdir
      - ./aks:/workspace/aks:rwx
    command: >
      mlflow server --serve-artifacts --host 0.0.0.0 --port 5000  
      --backend-store-uri "${DB_CONNECTION_STRING}" 
      --default-artifact-root s3://mlflow/ --expose-prometheus /prometheus

  minio:
    image: minio/minio:${MINIO_TAG}
    container_name: ${PROJECT_NAME}-minio
    hostname: minio
    restart: always
    stop_grace_period: 10s
    working_dir: /data
    volumes:
      - minio-data:/data
    ports:
      - 9000:9000
      - 9001:9001
    environment:
      MINIO_ACCESS_KEY: $DB_USER
      MINIO_SECRET_KEY: $DB_PASSWORD
      MINIO_ROOT_USER: $DB_USER
      MINIO_ROOT_PASSWORD: $DB_PASSWORD
      MINIO_STORAGE_USE_HTTPS: "False"
      MINIO_PROMETHEUS_AUTH_TYPE: public
    command: server --console-address ':9001' --address ':9000' /data 

  mc:
    image: minio/mc:${MINIO_MC_TAG}
    container_name: ${PROJECT_NAME}-mc
    hostname: mc
    environment:
      MINIO_PORT: 9000
      MINIO_ACCESS_KEY: $DB_USER
      MINIO_SECRET_KEY: $DB_PASSWORD
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c ' sleep 5;  /usr/bin/mc config host add s3  http://minio:9000 ${DB_USER} ${DB_PASSWORD} --api S3v4;   
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/mlflow;  /usr/bin/mc policy download s3/mlflow;   
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/bento;  /usr/bin/mc policy download s3/bento;   
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/data;  /usr/bin/mc policy download s3/data; exit 0; '

  postgres:
    image: postgres:${POSTGRES_TAG}
    container_name: ${PROJECT_NAME}-postgres
    hostname: postgres
    restart: always
    stop_grace_period: 10s
    environment:
      POSTGRES_PASSWORD: $DB_PASSWORD
      POSTGRES_DB: $DB_NAME
      POSTGRES_USER: $DB_USER
    ports:
      - 5432:5432
    volumes:
      - postgres-data:/var/lib/postgresql/data

  adminer:
    image: adminer:${ADMINER_TAG}
    container_name: ${PROJECT_NAME}-adminer
    hostname: adminer
    restart: always
    stop_grace_period: 10s
    ports:
      - 8081:8080

  zookeeper:
    image: confluentinc/cp-zookeeper:${KAFKA_TAG}
    container_name: ${PROJECT_NAME}-zookeeper
    hostname: zookeeper
    restart: unless-stopped
    stop_grace_period: 10s
    env_file: .env
    environment:
      ZOOKEEPER_CLIENT_PORT: 32181
    ports:
      - "32181:32181"

  kafka:
    image: confluentinc/cp-kafka:${KAFKA_TAG}
    container_name: ${PROJECT_NAME}-kafka
    hostname: kafka
    restart: unless-stopped
    stop_grace_period: 10s
    env_file: .env
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: true
    ports:
      - 9092:9092
    depends_on:
      - zookeeper

  spark:
    image: docker.io/bitnami/spark:${SPARK_TAG}
    container_name: ${PROJECT_NAME}-spark
    hostname: spark
    restart: unless-stopped
    stop_grace_period: 10s
    env_file: .env
    environment:
      SPARK_MODE: master
      SPARK_RPC_AUTHENTICATION_ENABLED: no
      SPARK_RPC_ENCRYPTION_ENABLED: no
      SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED: no
      SPARK_SSL_ENABLED: no
      SPARK_USER: spark
    ports:
      - 8080:8080
      - 7077:7077

  spark-worker:
    image: docker.io/bitnami/spark:${SPARK_TAG}
    container_name: ${PROJECT_NAME}-spark-worker
    hostname: spark_worker
    restart: unless-stopped
    stop_grace_period: 10s
    env_file: .env
    environment:
      SPARK_MODE: worker
      SPARK_MASTER_URL: spark://spark:7077
      SPARK_WORKER_MEMORY: 2G
      SPARK_WORKER_CORES: 1
      SPARK_RPC_AUTHENTICATION_ENABLED: no
      SPARK_RPC_ENCRYPTION_ENABLED: no
      SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED: no
      SPARK_SSL_ENABLED: no
      SPARK_USER: spark

  redis:
    image: redis:${REDIS_TAG}
    container_name: ${PROJECT_NAME}-redis
    hostname: redis
    restart: unless-stopped
    stop_grace_period: 10s
    env_file: .env
    ports:
      - 6379:6379

  flower:
    <<: *airflow-common
    container_name: ${PROJECT_NAME}-flower
    hostname: flower
    command: >
      celery --broker=redis://redis:6379/0 flower --port=5555 
    restart: unless-stopped
    stop_grace_period: 10s
    env_file: .env
    ports:
      - 5555:5555

networks:
  default:
    name: $PROJECT_NAME
    driver: bridge

volumes:
  postgres-data: {}
  minio-data: {}
  mlflow-prometheus-data: {}