docker-compose.yml

version: '3.3'

networks:

  revproxy:
    external:
      name: boot_revproxy

services:

  zeppelin:
    image: registry.gitlab.com/rychly-edu/docker/docker-zeppelin
    networks:
      - revproxy
    environment:
      - PROP_ZEPPELIN_zeppelin_server_addr=0.0.0.0
      - ZEPPELIN_PORT=8080
      - ZEPPELIN_NOTEBOOK_DIR=/home/zeppelin
      - MASTER=yarn
      - RESOURCE_MANAGER=resourcemanager
      - HISTUI_PORT=18080
      - LOG_DIR=hdfs://namenode:8020/user/spark/applicationHistory
      - HDFS_URL=http://namenode:9870/webhdfs/v1/
      - PROP_SPARK_spark_jars=/opt/michal-beder-tarzan/Java/ndx-spark-model/target,/opt/michal-beder-tarzan/Java/ndx-spark-pcap/target,/opt/michal-beder-tarzan/Java/ndx-spark-shell/target,/opt/michal-beder-tarzan/Java/ndx-spark-tshark/target
#    ports:
#      - "8080:8080"
    volumes:
      - ./volumes/notebook:/home/zeppelin
      - ./components/michal-beder-tarzan:/opt/michal-beder-tarzan
    labels:
      - "traefik.backend=zeppelin"
      - "traefik.port=8080"
      - "traefik.frontend.rule=PathPrefixStrip:/zeppelin/"
      - "traefik.docker.network=tarzan-platform-docker_revproxy"
    deploy:
      replicas: 1

  resourcemanager:
    image: registry.gitlab.com/rychly-edu/docker/docker-yarn
    networks:
      - revproxy
    environment:
      - ROLE=resourcemanager
      - RESOURCE_MANAGER=http://resourcemanager:8088,https://resourcemanager:8090,rpc://resourcemanager:8032,schrpc://resourcemanager:8030,rtrpc://resourcemanager:8031,admrpc://resourcemanager:8033
      - DFS_DEFAULT=hdfs://namenode:8020
#    ports:
#      - "8088:8088"
#      - "8090:8090"
#      - "8032:8032"
#      - "8030:8030"
#      - "8031:8031"
#      - "8033:8033"
    labels:
      - "traefik.backend=yarn"
      - "traefik.port=8088"
      - "traefik.frontend.rule=PathPrefixStrip:/yarn/"
      - "traefik.docker.network=tarzan-platform-docker_revproxy"
    deploy:
      placement:
        constraints: [node.role == manager]

  nodemanager:
    image: registry.gitlab.com/rychly-edu/docker/docker-yarn
    networks:
      - revproxy
    environment:
      - ROLE=nodemanager
      - NODE_MANAGER=http://nodemanager:8042,https://nodemanager:8044,locipc://nodemanager:8040,csipc://nodemanager:8048
      - RESOURCE_MANAGER=resourcemanager
      - DFS_DEFAULT=hdfs://namenode:8020
#    ports:
#      - "8042"
#      - "8044"
#      - "8040"
#      - "8048"
    deploy:
      mode: global

  sparkhistory:
    image: registry.gitlab.com/rychly-edu/docker/docker-spark
    networks:
      - revproxy
    environment:
      - ROLE=history
      - HISTUI_PORT=18080
      - LOG_DIR=hdfs://namenode:8020/user/spark/applicationHistory
      - WAIT_FOR=namenode:8020
#    ports:
#      - "18080:18080"
    labels:
      - "traefik.backend=spark"
      - "traefik.port=18080"
      - "traefik.frontend.rule=PathPrefixStrip:/spark/"
      - "traefik.docker.network=tarzan-platform-docker_revproxy"
    deploy:
      replicas: 1

  namenode:
    image: registry.gitlab.com/rychly-edu/docker/docker-hdfs
    networks:
      - revproxy
    environment:
      - ROLE=namenode
      - DFS_DEFAULT=hdfs://namenode:8020
      - NAME_DIRS=/home/hadoop/name
      - DFS_NAMENODES=rpc://namenode:8020,http://namenode:9870,https://namenode:9871
      - ADD_USERS=spark,zeppelin
#    ports:
#      - "8020:8020"
#      - "9870:9870"
#      - "9871:9871"
    volumes:
      - ./volumes/namenode:/home/hadoop/name
    labels:
      - "traefik.backend=hdfs"
      - "traefik.port=9870"
      - "traefik.frontend.rule=PathPrefixStrip:/hdfs/"
      - "traefik.docker.network=tarzan-platform-docker_revproxy"
    deploy:
      placement:
        constraints: [node.role == manager]

  datanode:
    image: registry.gitlab.com/rychly-edu/docker/docker-hdfs
    networks:
      - revproxy
    environment:
      - ROLE=datanode
      - DFS_DEFAULT=hdfs://namenode:8020
      - DATA_DIRS=/home/hadoop/data
      - DFS_DATANODES=data://0.0.0.0:9866,http://0.0.0.0:9864,https://0.0.0.0:9865,ipc://0.0.0.0:9867
#    ports:
#      - "9866"
#      - "9864"
#      - "9865"
#      - "9867"
    volumes:
      - ./volumes/datanode1:/home/hadoop/data
    deploy:
      mode: global

#  revproxy:
#    image: traefik
#    networks:
#      - revproxy
#    command:
#      - "--api"
#      - "--entrypoints=Name:http Address::80 Redirect.EntryPoint:https"
#      - "--entrypoints=Name:https Address::443 TLS"
#      - "--defaultentrypoints=http,https"
#      - "--acme"
#      # Frequent restarts of the service/certificate requests will be blocked by LE servers as DoS attacks, unless the ACME storage is persistent (in a Docker volume).
#      - "--acme.storage=acme.json"
#      - "--acme.entryPoint=https"
#      - "--acme.httpChallenge.entryPoint=http"
#      - "--acme.onHostRule=true"
#      - "--acme.email=user@hostname.domain"
#      #- "--acme.acmelogging=true"
#      - "--acme.domains=hostname.domain"
#      - "--docker"
#      # SWARM only
#      #- "--docker.swarmMode"
#      - "--docker.domain=hostname.domain"
#      - "--docker.watch"
#      - "--logLevel=INFO"
#    volumes:
#      - type: bind
#        source: /var/run/docker.sock
#        target: /var/run/docker.sock
#    ports:
#      - target: 80
#        published: 8080
#        mode: host
#      - target: 443
#        published: 8443
#        mode: host
#    labels:
#      - "traefik.backend=revproxy"
#      - "traefik.port=8080"
#      - "traefik.frontend.rule=PathPrefixStrip:/traefik/"
#      - "traefik.docker.network=tarzan-platform-docker_revproxy"
#    deploy:
#      mode: global
#      placement:
#        constraints: [node.role == manager]
#      update_config:
#        parallelism: 1
#        delay: 10s
#      restart_policy:
#        condition: on-failure