compose.yml

# Common build configuration for Airflow
# Extension field, see https://docs.docker.com/compose/compose-file/compose-file-v3/#extension-fields
x-airflow-common: &airflow-common
  profiles:
    - catalog
  restart: on-failure
  depends_on:
    - postgres
    - s3
  image: openverse_catalog
  env_file:
    - .env
  build:
    target: cat
    args: # Automatically inferred from env vars, unless specified
      - REQUIREMENTS_FILE=requirements-dev.txt
      - CATALOG_PY_VERSION
      - CATALOG_AIRFLOW_VERSION
  volumes:
    - .:/opt/airflow/catalog:z
    - catalog-cache:/home/airflow/.cache

services:
  # Dev changes for the scheduler
  scheduler:
    <<: *airflow-common
    depends_on:
      - upstream_db
      - s3
    command: scheduler
    expose:
      - "8793" # Used for fetching logs
    environment:
      # Upgrade the DB on startup
      _AIRFLOW_DB_MIGRATE: "true"
      _AIRFLOW_WWW_USER_CREATE: "true"
      _AIRFLOW_WWW_USER_USERNAME: airflow
      _AIRFLOW_WWW_USER_PASSWORD: airflow
      _AIRFLOW_WWW_USER_FIRSTNAME: Air
      _AIRFLOW_WWW_USER_LASTNAME: Flow
      _AIRFLOW_WWW_USER_EMAIL: airflow@example.com

  # Dev changes for the triggerer
  triggerer:
    <<: *airflow-common
    depends_on:
      - upstream_db
      - s3
    expose:
      - "8794" # Used for logs
    command: triggerer

  # Dev changes for the webserver container
  webserver:
    <<: *airflow-common
    depends_on:
      - upstream_db
      - s3
      - scheduler
      - triggerer
    command: webserver
    ports:
      - "${AIRFLOW_PORT}:8080"

  s3:
    profiles:
      - catalog_dependencies
      - catalog
    image: docker.io/minio/minio:latest
    ports:
      - "5010:5000"
      - "5011:5001"
    env_file:
      - .env
      - ../docker/minio/.env
    command: minio server /data --address :5000 --console-address :5001
    volumes:
      - minio:/data
      - ../docker/minio/s3_entrypoint.sh:/opt/minio/s3_entrypoint.sh:ro,z
    entrypoint: /opt/minio/s3_entrypoint.sh
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:5010/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  load_to_s3:
    profiles:
      - catalog_dependencies
      - catalog
    image: docker.io/minio/mc:latest
    env_file:
      - .env
      - ../docker/minio/.env
    depends_on:
      - s3
    volumes:
      # Buckets for testing provider data imported from s3 are subdirectories under
      # /tests/s3-data/
      - ./tests/s3-data:/data:rw,z
      - ../docker/minio/load_to_s3_entrypoint.sh:/opt/minio/load_to_s3_entrypoint.sh:ro,z
    entrypoint: /opt/minio/load_to_s3_entrypoint.sh

volumes:
  catalog-cache:
  minio: