docker-compose.test.yml

# -*- coding: utf-8 -*-
#
# This file is part of hepcrawl.
# Copyright (C) 2017 CERN.
#
# hepcrawl is a free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

version: '2.1'

services:
  service_base: &service_base
    image: hepcrawl_base  # hepcrawl_base image is build at pip service of docker-compose.deps.yml
    environment:
      - APP_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
      - APP_CELERY_RESULT_BACKEND=amqp://guest:guest@rabbitmq:5672//
      - APP_CRAWLER_HOST_URL=http://scrapyd:6800
      - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results
      - APP_FILES_STORE=/tmp/file_urls
      - APP_CRAWL_ONCE_PATH=/code/.scrapy
      - COVERAGE_PROCESS_START=/code/.coveragerc
      - BASE_USER_UID=${BASE_USER_UID:-1000}
      - BASE_USER_GIT=${BASE_USER_GIT:-1000}
    tty: true
    volumes:
      - ${DOCKER_DATA}/tmp/hepcrawl_venv:/hepcrawl_venv/
      - ${PWD}:/code/
      - ${PWD}/tests/functional/scrapyd_coverage_runner.conf:/etc/scrapyd/scrapyd.conf
      - /tmp/WSP:/tmp/WSP
      - /tmp/file_urls:/tmp/file_urls

  functional_wsp:
    <<: *service_base
    command: py.test -vv tests/functional/wsp
    depends_on:
      scrapyd:
        condition: service_healthy
      ftp_server:
        condition: service_healthy

  functional_desy:
    <<: *service_base
    command: py.test -vv tests/functional/desy
    depends_on:
      scrapyd:
        condition: service_healthy
      ftp_server:
        condition: service_healthy

  functional_arxiv:
    <<: *service_base
    command: py.test -vv tests/functional/arxiv
    depends_on:
      scrapyd:
        condition: service_healthy

  functional_cds:
    <<: *service_base
    command: py.test -vv tests/functional/cds
    links:
      - scrapyd

  functional_pos:
    <<: *service_base
    command: py.test -vv tests/functional/pos
    depends_on:
      scrapyd:
        condition: service_healthy
      http-server.local:
        condition: service_healthy

  unit:
    <<: *service_base
    command: bash -c "py.test tests/unit -vv && make -C docs clean && make -C docs html && python setup.py sdist && ls dist/*"
    links: []

  celery:
    <<: *service_base
    command: celery worker --events --app hepcrawl.testlib.tasks --loglevel=debug
    depends_on:
      rabbitmq:
        condition: service_healthy

  scrapyd:
    <<: *service_base
    command: bash -c "rm -f twistd.pid && exec scrapyd"
    depends_on:
      celery:
        condition: service_started
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test:
        - "CMD-SHELL"
        - "curl http://localhost:6800/listprojects.json"

  scrapyd-deploy:
    <<: *service_base
    command: bash -c "scrapyd-deploy"
    depends_on:
      scrapyd:
        condition: service_healthy

  ftp_server:
    image: stilliard/pure-ftpd:hardened
    environment:
      - PUBLICHOST=localhost
    volumes:
      - ${PWD}/tests/functional/desy/fixtures/ftp_server/FFT:/home/ftpusers/bob/FFT
      - ${PWD}/tests/functional/desy/fixtures/ftp_server/DESY:/home/ftpusers/bob/DESY
      - ${PWD}/tests/functional/wsp/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP
      - ${PWD}/tests/functional/wsp/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd

  http-server.local:
    image: nginx:stable-alpine
    volumes:
      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
      - ${PWD}/tests/functional/pos/fixtures/https_server/conf/ssl:/etc/nginx/ssl
      - ${PWD}/tests/functional/pos/fixtures/https_server/records:/etc/nginx/html/
    ports:
      - 443:443
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test:
        - "CMD-SHELL"
        - "curl https://localhost:443/"

  rabbitmq:
    image: rabbitmq
    healthcheck:
      timeout: 5s
      interval: 5s
      retries: 5
      test:
        - "CMD"
        - "rabbitmqctl"
        - "status"