docker-compose.test.py3.yml

# -*- coding: utf-8 -*-
#
# This file is part of hepcrawl.
# Copyright (C) 2017 CERN.
#
# hepcrawl is a free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

version: '2.1'

services:
  service_base: &service_base
    image: hepcrawl_py3
    environment:
      - APP_BROKER_URL=pyamqp://guest:guest@rabbitmq:5672//
      - APP_CELERY_RESULT_BACKEND=redis://redis:6379/1
      - APP_CRAWLER_HOST_URL=http://scrapyd:6800
      - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results
      - APP_FILES_STORE=/tmp/file_urls
      - APP_LAST_RUNS_PATH=/code/.scrapy/last_runs
      - APP_CRAWL_ONCE_PATH=/code/.scrapy
      - COVERAGE_PROCESS_START=/code/.coveragerc
    tty: true
    volumes:
      - .:/code/
    user: "${UID:-1000}:${GID:-1000}"

  functional_wsp:
    <<: *service_base
    command: py.test -vv tests/functional/wsp
    depends_on:
      scrapyd:
        condition: service_healthy
      ftp_server:
        condition: service_healthy

  functional_desy:
    <<: *service_base
    command: py.test -vv tests/functional/desy
    depends_on:
      scrapyd:
        condition: service_healthy
      ftp_server:
        condition: service_healthy

  functional_arxiv:
    <<: *service_base
    command: py.test -vv tests/functional/arxiv
    depends_on:
      scrapyd:
        condition: service_healthy
      arxiv-http-server.local:
        condition: service_healthy

  functional_pos:
    <<: *service_base
    command: py.test -vv tests/functional/pos
    depends_on:
      scrapyd:
        condition: service_healthy
      http-server.local:
        condition: service_healthy

  unit:
    <<: *service_base
    command: py.test -vv tests/unit
    links: []

  celery:
    <<: *service_base
    command: celery worker --events --app hepcrawl.testlib.tasks --loglevel=debug
    depends_on:
      rabbitmq:
        condition: service_healthy
      redis:
        condition: service_healthy

  scrapyd:
    <<: *service_base
    command: scrapyd --pidfile=/tmp/scrapyd.pid
    networks:
      default:
      ftp:
    depends_on:
      celery:
        condition: service_started
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k http://localhost:6800/listprojects.json"

  scrapyd-deploy:
    <<: *service_base
    command: "true"  # do nothing, as the real thing currently fails on Python 3, and not needed for unit
    # command: bash -c "scrapyd-deploy"
    # depends_on:
    #   scrapyd:
    #     condition: service_healthy

  ftp_server:
    image: stilliard/pure-ftpd:hardened
    environment:
      - PUBLICHOST=1.2.3.4
    networks:
      ftp:
        ipv4_address: 1.2.3.4
    volumes:
      - ${PWD}/tests/functional/desy/fixtures/ftp_server/FFT:/home/ftpusers/bob/FFT
      - ${PWD}/tests/functional/desy/fixtures/ftp_server/DESY:/home/ftpusers/bob/DESY
      - ${PWD}/tests/functional/wsp/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP
      - ${PWD}/tests/functional/wsp/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "ls -l /var/run/pure-ftpd.pid"

  http-server.local:
    image: nginx:stable-alpine
    volumes:
      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/ssl:/etc/nginx/ssl
      - ${PWD}/tests/functional/pos/fixtures/https_server/records:/etc/nginx/html/
    ports:
      - 443:443
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k https://localhost:443/187.html"

  functional_cds:
    <<: *service_base
    command: py.test -vv tests/functional/cds
    depends_on:
      scrapyd:
        condition: service_healthy
      cds-http-server.local:
        condition: service_healthy

  arxiv-http-server.local:
    image: nginx:stable-alpine
    volumes:
      - ${PWD}/tests/functional/arxiv/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/arxiv/fixtures/http_server/records:/etc/nginx/html/
    ports:
      - 80:80
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k http://localhost:80/arxiv-physics-hep-th.xml"

  cds-http-server.local:
    image: nginx:stable-alpine
    volumes:
      - ${PWD}/tests/functional/cds/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/cds/fixtures/http_server/records:/etc/nginx/html/
    ports:
      - 80:80
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k http://localhost:80/cds-single.xml"

  rabbitmq:
    image: rabbitmq
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "rabbitmqctl status"

  redis:
    image: redis:3.2.3
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "redis-cli -h 127.0.0.1 ping| grep PONG"

  localstack:
    image: localstack/localstack:latest
    ports:
      - '4572:4572'
      - '4566:4566'
    environment:
      - SERVICES=s3
      - DEBUG=1
      - DATA_DIR=/home/localstack/data
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test: "curl -k localhost:4566|grep running"

networks:
  ftp:
    ipam:
      config:
        - subnet: 1.0.0.0/8