docker-compose.yml

version: "3.7"
services:
  spark-master:
    image: spark-master:2.4.3
    container_name: spark_master
    hostname: spark-master
    ports:
      - "8088:8080"
      - "7078:7077"
    networks: 
      spark_network:
    volumes:
       - /mnt/spark-apps:/mnt/spark-apps
       - /mnt/spark-data:/mnt/spark-data
    environment:
      - "SPARK_LOCAL_IP=spark-master"
  spark-worker-1:
    image: spark-worker:2.4.3
    container_name: my_spark_worker_1
    hostname: spark-worker-1
    depends_on:
      - spark-master
    ports:
      - "8081:8081"
    env_file: ./env/spark-worker.sh
    environment:
      - "SPARK_LOCAL_IP=spark-worker-1"
    networks: 
      spark_network:
    volumes:
       - /mnt/spark-apps:/mnt/spark-apps
       - /mnt/spark-data:/mnt/spark-data
  spark-worker-2:
    image: spark-worker:2.4.3
    container_name: spark_worker_2
    hostname: spark-worker-2
    depends_on:
      - spark-master
    ports:
      - "8082:8081"
    env_file: ./env/spark-worker.sh
    environment:
      - "SPARK_LOCAL_IP=spark-worker-2"
    networks: 
      spark_network:
    volumes:
       - /mnt/spark-apps:/mnt/spark-apps
       - /mnt/spark-data:/mnt/spark-data

networks:
  spark_network:
    driver: bridge
    ipam:
     driver: default