Add Production-ready docker compose for the production image

[potiuk]

Description

In order to use the production image we are already working on a helm chart, but we might want to add a production-ready docker compose that will be able to run airflow installation.

Use case / motivation

For local tests/small deployments - being able to have such docker-compose environment would be really nice.

We seem to get to consensus that we need to have several docker-compose "sets" of files:

• Local Executor
• Celery Executor
• Kubernetes Executor (??? do we need to have a Kubernetes Executor in a Compose ? I guess not...)

They should be varianted and possible to specify the number of parameters:

• Database (Postgres/MySQL)
• Redis vs. Rabitmq (should we choose one ???)
• Ports
• Volumes (persistent / not)
• Airflow Images
• Fernet Key
• RBAC

Depending on the setup, those Docker compose file should do proper DB initialisation.

---

Example Docker Compose (From https://apache-airflow.slack.com/archives/CQAMHKWSJ/p1587748008106000) that we might use as a base and #8548 . This is just example so this issue will not implement all of it and we will likely split those docker-compose into separate postgres/sqlite/mysql similarly as we do in CI script, so I wanted to keep it as separate issue - we will deal with user creation in #8606

version: '3'
services:
  postgres:
    image: postgres:latest
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres
      - POSTGRES_DB=airflow
      - POSTGRES_PORT=5432
    ports:
      - 5432:5432
  redis:
    image: redis:latest
    ports:
      - 6379:6379
  flower:
    image: apache/airflow:1.10.10
    volumes:
      - ./airflow-data/dags:/opt/airflow/dags
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__RBAC=True
    command: flower
    ports:
      - 5555:5555
  airflow:
    image: apache/airflow:1.10.10
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__RBAC=True
    command: webserver
    ports:
      - 8080:8080
    volumes:
      - ./airflow-data/dags:/opt/airflow/dags
      - ./airflow-data/logs:/opt/airflow/logs
      - ./airflow-data/plugins:/opt/airflow/plugins
  airflow-scheduler:
    image: apache/airflow:1.10.10
    container_name: airflow_scheduler_cont
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__RBAC=True
    command: scheduler
    volumes:
      - ./airflow-data/dags:/opt/airflow/dags
      - ./airflow-data/logs:/opt/airflow/logs
      - ./airflow-data/plugins:/opt/airflow/plugins
  airflow-worker1:
    image: apache/airflow:1.10.10
    container_name: airflow_worker1_cont
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__RBAC=True
    command: worker
    volumes:
      - ./airflow-data/dags:/opt/airflow/dags
      - ./airflow-data/logs:/opt/airflow/logs
      - ./airflow-data/plugins:/opt/airflow/plugins
  airflow-worker2:
    image: apache/airflow:1.10.10
    container_name: airflow_worker2_cont
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__RBAC=True
    command: worker
    volumes:
      - ./airflow-data/dags:/opt/airflow/dags
      - ./airflow-data/logs:/opt/airflow/logs
      - ./airflow-data/plugins:/opt/airflow/plugins
  airflow-worker3:
    image: apache/airflow:1.10.10
    container_name: airflow_worker3_cont
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow
      - AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__RBAC=True
    command: worker
    volumes:
      - ./airflow-data/dags:/opt/airflow/dags
      - ./airflow-data/logs:/opt/airflow/logs
      - ./airflow-data/plugins:/opt/airflow/plugins

Another example from https://apache-airflow.slack.com/archives/CQAMHKWSJ/p1587679356095400:

version: '3.7'
networks:
  airflow:
    name: airflow
    attachable: true
volumes:
  logs:
x-database-env: 
  &database-env
  POSTGRES_USER: airflow
  POSTGRES_DB: airflow
  POSTGRES_PASSWORD: airflow
x-airflow-env: 
  &airflow-env
  AIRFLOW__CORE__EXECUTOR: CeleryExecutor
  AIRFLOW__WEBSERVER__RBAC: 'True'
  AIRFLOW__CORE__CHECK_SLAS: 'False'
  AIRFLOW__CORE__STORE_SERIALIZED_DAGS: 'False'
  AIRFLOW__CORE__PARALLELISM: 50
  AIRFLOW__CORE__LOAD_EXAMPLES: 'False'
  AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: 'False'
  AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC: 10
  
services:
  postgres:
    image: postgres:11.5
    environment:
      <<: *database-env
      PGDATA: /var/lib/postgresql/data/pgdata
    ports:
      - 5432:5432
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./database/data:/var/lib/postgresql/data/pgdata
      - ./database/logs:/var/lib/postgresql/data/log
    command: >
     postgres
       -c listen_addresses=*
       -c logging_collector=on
       -c log_destination=stderr
       -c max_connections=200
    networks:
      - airflow
  redis:
    image: redis:5.0.5
    environment:
      REDIS_HOST: redis
      REDIS_PORT: 6379
    ports:
      - 6379:6379
    networks:
      - airflow
  webserver:
    image: airflow:1.10.10
    user: airflow
    ports:
      - 8090:8080
    volumes:
      - ./dags:/opt/airflow/dags
      - logs:/opt/airflow/logs
      - ./files:/opt/airflow/files
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      <<: *database-env
      <<: *airflow-env
      ADMIN_PASSWORD: airflow
    depends_on:
      - postgres
      - redis
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3
    networks:
      - airflow
  flower:
    image: airflow:1.10.10
    user: airflow
    ports:
      - 5555:5555
    depends_on:
      - redis
    volumes:
      - logs:/opt/airflow/logs
    command: flower
    networks:
      - airflow
  scheduler:
    image: airflow:1.10.10
    volumes:
      - ./dags:/opt/airflow/dags
      - logs:/opt/airflow/logs
      - ./files:/opt/airflow/files
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      <<: *database-env
    command: scheduler
    networks:
      - airflow
  worker:
    image: airflow:1.10.10
    user: airflow
    volumes:
      - ./dags:/opt/airflow/dags
      - logs:/opt/airflow/logs
      - ./files:/opt/airflow/files
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      <<: *database-env
    command: worker
    depends_on:
      - scheduler

Related issues
The initial user creation #8606, #8548
Quick start documentation planned in #8542

[potiuk]

This one duplicates #8548 a bit - but I want to leave it for a while as I wanted to split it into smaller functional pieces.

[kaxil]

It would be nice to have this in "Quick Start Guide when using Docker Image" too. WDYT

[potiuk]

Absolutely. It's already planned in #8542 :)

[potiuk]

Added missing label :)

[habibdhif]

Here is another example of a Docker Compose that I've been working on. The Compose defines multiple services to run Airflow.
There is an init service which is an ephemeral container to initialize the database and creates a user if necessary.
The init service command tries to run airflow list_users and if it fails it initializes the database and creates a user. Different approaches were considered but this one is simple enough and only involves airflow commands (no database-specific commands).

Extension fields are used for airflow environment variables to reduce code duplication.

I added a Makefile along the docker-compose.yml in my repo so all you have to do to run the docker-compose is run make run.

version: "3.7"
x-airflow-environment: &airflow-environment
  AIRFLOW__CORE__EXECUTOR: CeleryExecutor
  AIRFLOW__WEBSERVER__RBAC: "True"
  AIRFLOW__CORE__LOAD_EXAMPLES: "False"
  AIRFLOW__CELERY__BROKER_URL: "redis://:@redis:6379/0"
  AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres:5432/airflow

services:
  postgres:
    image: postgres:11.5
    environment:
      POSTGRES_USER: airflow
      POSTGRES_DB: airflow
      POSTGRES_PASSWORD: airflow
  redis:
    image: redis:5
    environment:
      REDIS_HOST: redis
      REDIS_PORT: 6379
    ports:
      - 6379:6379
  init:
    image: apache/airflow:1.10.10
    environment:
      <<: *airflow-environment
    depends_on:
      - redis
      - postgres
    volumes:
      - ./dags:/opt/airflow/dags
    entrypoint: /bin/bash
    command: >
      -c "airflow list_users || (airflow initdb
      && airflow create_user --role Admin --username airflow --password airflow -e airflow@airflow.com -f airflow -l airflow)"
    restart: on-failure
  webserver:
    image: apache/airflow:1.10.10
    ports:
      - 8080:8080
    environment:
      <<: *airflow-environment
    depends_on:
      - init
    volumes:
      - ./dags:/opt/airflow/dags
    command: "webserver"
    restart: always
  flower:
    image: apache/airflow:1.10.10
    ports:
      - 5555:5555
    environment:
      <<: *airflow-environment
    depends_on:
      - redis
    command: flower
    restart: always
  scheduler:
    image: apache/airflow:1.10.10
    environment:
      <<: *airflow-environment
    depends_on:
      - webserver
    volumes:
      - ./dags:/opt/airflow/dags
    command: scheduler
    restart: always
  worker:
    image: apache/airflow:1.10.10
    environment:
      <<: *airflow-environment
    depends_on:
      - scheduler
    volumes:
      - ./dags:/opt/airflow/dags
    command: worker
    restart: always

[infused-kim]

Here's my docker-compose config using LocalExecutor...

docker-compose.airflow.yml:

version: '2.1'
services:
    airflow:
        # image: apache/airflow:1.10.10
        build:
            context: .
            args:
                - DOCKER_UID=${DOCKER_UID-1000} 
            dockerfile: Dockerfile
        restart: always
        environment:
            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgres://airflow:${POSTGRES_PW-airflow}@postgres:5432/airflow
            - AIRFLOW__CORE__FERNET_KEY=${AF_FERNET_KEY-GUYoGcG5xdn5K3ysGG3LQzOt3cc0UBOEibEPxugDwas=}
            - AIRFLOW__CORE__EXECUTOR=LocalExecutor
            - AIRFLOW__CORE__AIRFLOW_HOME=/opt/airflow/
            - AIRFLOW__CORE__LOAD_EXAMPLES=False
            - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
            - AIRFLOW__CORE__LOGGING_LEVEL=${AF_LOGGING_LEVEL-info}
        volumes:
            - ../airflow/dags:/opt/airflow/dags:z
            - ../airflow/plugins:/opt/airflow/plugins:z
            - ./volumes/airflow_data_dump:/opt/airflow/data_dump:z
            - ./volumes/airflow_logs:/opt/airflow/logs:z
        healthcheck:
            test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"]
            interval: 30s
            timeout: 30s
            retries: 3

docker-compose.yml:

version: '2.1'
services:
    postgres:
        image: postgres:9.6
        container_name: af_postgres
        environment:
            - POSTGRES_USER=airflow
            - POSTGRES_PASSWORD=${POSTGRES_PW-airflow}
            - POSTGRES_DB=airflow
            - PGDATA=/var/lib/postgresql/data/pgdata
        volumes:
            - ./volumes/postgres_data:/var/lib/postgresql/data/pgdata:Z
        ports:
            -  127.0.0.1:5432:5432

    webserver:
        extends:
            file: docker-compose.airflow.yml
            service: airflow
        container_name: af_webserver
        command: webserver
        depends_on:
            - postgres
        ports:
            - ${DOCKER_PORTS-8080}
        networks:
            - proxy
            - default
        environment:
            # Web Server Config
            - AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW=graph
            - AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT=true
            - AIRFLOW__WEBSERVER__RBAC=true

            # Web Server Performance tweaks
            # 2 * NUM_CPU_CORES + 1
            - AIRFLOW__WEBSERVER__WORKERS=${AF_WORKERS-2}
            # Restart workers every 30min instead of 30seconds
            - AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL=1800
        labels:
            - "traefik.enable=true"
            - "traefik.http.routers.airflow.rule=Host(`af.example.com`)"
            - "traefik.http.routers.airflow.middlewares=admin-auth@file"

    scheduler:
        extends:
            file: docker-compose.airflow.yml
            service: airflow
        container_name: af_scheduler
        command: scheduler
        depends_on:
            - postgres
        environment:
            # Performance Tweaks
            # Reduce how often DAGs are reloaded to dramatically reduce CPU use
            - AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL=${AF_MIN_FILE_PROCESS_INTERVAL-60} 
            - AIRFLOW__SCHEDULER__MAX_THREADS=${AF_THREADS-1}

networks:
    proxy:
        external: true

Dockerfile:

# Custom Dockerfile
FROM apache/airflow:1.10.10

# Install mssql support & dag dependencies
USER root
RUN apt-get update -yqq \
    && apt-get install -y gcc freetds-dev \
    && apt-get install -y git procps \ 
    && apt-get install -y vim
RUN pip install apache-airflow[mssql,mssql,ssh,s3,slack] 
RUN pip install azure-storage-blob sshtunnel google-api-python-client oauth2client \
    && pip install git+https://github.com/infusionsoft/Official-API-Python-Library.git \
    && pip install rocketchat_API

# This fixes permission issues on linux. 
# The airflow user should have the same UID as the user running docker on the host system.
# make build is adjust this value automatically
ARG DOCKER_UID
RUN \
    : "${DOCKER_UID:?Build argument DOCKER_UID needs to be set and non-empty. Use 'make build' to set it automatically.}" \
    && usermod -u ${DOCKER_UID} airflow \
    && find / -path /proc -prune -o -user 50000 -exec chown -h airflow {} \; \
    && echo "Set airflow's uid to ${DOCKER_UID}"

USER airflow

Makefile

And here's my Makefile to control it the containers like make run:

SERVICE = "scheduler"
TITLE = "airflow containers"
ACCESS = "http://af.example.com"

.PHONY: run

build:
	docker-compose build

run:
	@echo "Starting $(TITLE)"
	docker-compose up -d
	@echo "$(TITLE) running on $(ACCESS)"

runf:
	@echo "Starting $(TITLE)"
	docker-compose up

stop:
	@echo "Stopping $(TITLE)"
	docker-compose down

restart: stop print-newline run

tty:
	docker-compose run --rm --entrypoint='' $(SERVICE) bash

ttyr:
	docker-compose run --rm --entrypoint='' -u root $(SERVICE) bash

attach:
	docker-compose exec $(SERVICE) bash

attachr:
	docker-compose exec -u root $(SERVICE) bash

logs:
	docker-compose logs --tail 50 --follow $(SERVICE)

conf:
	docker-compose config

initdb:
	docker-compose run --rm $(SERVICE) initdb

upgradedb:
	docker-compose run --rm $(SERVICE) upgradedb

print-newline:
	@echo ""
	@echo ""