webapp.yml

version: '3.2'
services:

  # https://gitlab.com/rychly-edu/docker/docker-spark-app/container_registry
  # http://localhost:54040/
  sparkapp:
    # hadoop 3.2.0 bug: missing lib/native/lihdfs.so, so cannot use pyarrow hdfs -> use hadoop 3.1.*
    image: registry.gitlab.com/rychly-edu/docker/docker-spark-app:spark2.4-hadoop3.1
    environment:
      - MASTER_URL=spark://sparkmaster:7077
      - SPARK_JARS=/app/lib
      - SPARK_PYFILES=/app/lib
      - SPARK_APP=/app/webapp_main.py
      - WEBUI_PORT=4040
      - PP_HDFS_URI=hdfs://hadoop@namenode:8020/test_data
      - PP_HOST=0.0.0.0
      - PP_PORT=54380
      - WAIT_FOR=namenode:8020
      - MAKE_HDFS_DIRS=hdfs://hadoop@namenode:8020/test_data?user=spark&perms=1777
    ports:
      - "54380:54380"
      - "54040:4040"
    volumes:
      - type: bind
        source: ./volumes/app
        target: /app
        read_only: true

  # https://gitlab.com/rychly-edu/docker/docker-spark/container_registry
  # http://localhost:54041/
  sparkmaster:
    image: registry.gitlab.com/rychly-edu/docker/docker-spark:2.4.4-hadoop3.1
    environment:
      - ROLE=master
      - MASTER_PORT=7077
      - WEBUI_PORT=4040
    ports:
#      - "7077:7077"
      - "54041:4040"

  # https://gitlab.com/rychly-edu/docker/docker-spark/container_registry
  # http://localhost:54042/
  sparkworker:
    # hadoop 3.2.0 bug: missing lib/native/lihdfs.so, so cannot use pyarrow hdfs -> use hadoop 3.1.*
    image: registry.gitlab.com/rychly-edu/docker/docker-spark:2.4.4-hadoop3.1
    # to restart lost workers (i.e., workers exited with after a task or on a failure when asked to kill an executor)
    restart: always
    environment:
      - ROLE=worker
      - MASTER_URL=spark://sparkmaster:7077
      - WEBUI_PORT=4040
      - INSTALL_PKGS=python-libfwsi python-libregf
    ports:
      - "54042:4040"

  # https://gitlab.com/rychly-edu/docker/docker-hdfs/container_registry
  # http://localhost:54987/
  namenode:
    image: registry.gitlab.com/rychly-edu/docker/docker-hdfs:3.1.3
    environment:
      - ROLE=namenode
      - DFS_DEFAULT=hdfs://namenode:8020
      - NAME_DIRS=/home/hadoop/name
      - DFS_NAMENODES=rpc://namenode:8020,http://namenode:9870,https://namenode:9871
      - ADD_USERS=spark,hbase
    ports:
#      - "8020:8020"
      - "54987:9870"
#      - "9871:9871"
    volumes:
      - ./volumes/namenode:/home/hadoop/name
  datanode:
    image: registry.gitlab.com/rychly-edu/docker/docker-hdfs:3.1.3
    environment:
      - ROLE=datanode
      - DFS_DEFAULT=hdfs://namenode:8020
      - DATA_DIRS=/home/hadoop/data
      - DFS_DATANODES=data://0.0.0.0:9866,http://0.0.0.0:9864,https://0.0.0.0:9865,ipc://0.0.0.0:9867
#    ports:
#      - "9866"
#      - "9864"
#      - "9865"
#      - "9867"
    volumes:
      - ./volumes/datanode1:/home/hadoop/data

  # http://localhost:58081/ta/
  timeline-analyzer:
    image: registry.gitlab.com/rychly-edu/docker/docker-timeline-analyzer
    environment:
      - NGINX_PORT=8081
      - RDF4J_SERVER_HOST=halyard-webapps
      - RDF4J_SERVER_PORT=8082
      - RDF4J_WORKBENCH_HOST=halyard-webapps
      - RDF4J_WORKBENCH_PORT=8082
    ports:
      - "58081:8081"

  # http://localhost:8082/
  halyard-webapps:
    image: registry.gitlab.com/rychly-edu/docker/docker-halyard-webapps:3.0-hadoop3.1-hbase2.2
    environment:
      - ZOO_SERVERS=zookeeper
      - ZOO_PORT=2181
      - ROOT_DIR=hdfs://namenode:8020/user/hbase
      - JETTY_PORT=8082
#    ports:
#      - "8082:8082"

  hbase-master:
    image: registry.gitlab.com/rychly-edu/docker/docker-hbase:2.2.0
    environment:
      - ROLE=master
      - ZOO_SERVERS=zookeeper
      - ZOO_PORT=2181
      - ROOT_DIR=hdfs://namenode:8020/user/hbase
      - WAIT_FOR=namenode:8020
      - MASTER_PORT=16000
      - MASTER_INFO_PORT=16010
#    ports:
#      - "16000:16000"
#      - "16010:16010"

  regionserver:
    image: registry.gitlab.com/rychly-edu/docker/docker-hbase:2.2.0
    environment:
      - ROLE=regionserver
      - ZOO_SERVERS=zookeeper
      - ZOO_PORT=2181
      - ROOT_DIR=hdfs://namenode:8020/user/hbase
      - WAIT_FOR=namenode:8020
      - REGIONSERVER_PORT=16020
      - REGIONSERVER_INFO_PORT=16030
#    ports:
#      - "16020:16020"
#      - "16030:16030"

  zookeeper:
    image: registry.gitlab.com/rychly-edu/docker/docker-hbase:2.2.0
    environment:
      - ROLE=zookeeper
      - ZOO_DATA_DIR=/home/hbase/zk-data
      - ZOO_DATA_LOG_DIR=/home/hbase/zk-data-log
      - ZOO_PORT=2181
#      - ZOO_MY_ID=1
#      - ZOO_SERVERS=localhost
#      - ZOO_PEERPORT=2888
#      - ZOO_LEADERPORT=3888
#    ports:
#      - "2181:2181"
    volumes:
      - ./volumes/zk-data:/home/hbase/zk-data
      - ./volumes/zk-data-log:/home/hbase/zk-data-log