docker-compose.yml

version: "3.7"
services:

  hive-metastore:
    image: 'naushadh/hive-metastore'
    container_name: hive-metastore
    hostname: hive-metastore
    ports:
      - '9083:9083' # Metastore Thrift
    environment:
      - DATABASE_HOST=postgresdb
      - DATABASE_DB=metastore_db
      - DATABASE_USER=admin
      - DATABASE_PASSWORD=admin
      - S3_PREFIX=myminio
      - S3_BUCKET=warehouse
      - S3_ENDPOINT_URL=http://minio:9000
      - AWS_ACCESS_KEY_ID=minioadmin
      - AWS_SECRET_ACCESS_KEY=minioadmin
    depends_on:
      - postgresdb
      - minio
    networks:
      - dbt_sample_network

  spark-thrift:
    image: spark-thrift
    container_name: spark-thrift
    build:
      context: .
      dockerfile: SparkThriftDockerfile
    ports:
      - "10000:10000"
      - "4040:4040"
    depends_on:
      - hive-metastore
    command: >
      --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
      --name Thrift JDBC/ODBC Server
      --hiveconf hive.server2.thrift.bind.host=0.0.0.0
    volumes:
      - ./.spark-warehouse/:/spark-warehouse/
      - ./config/hive-site.xml:/usr/spark/conf/hive-site.xml
      - ./config/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf
    environment:
      - WAIT_FOR=hive-metastore:9083
      - AWS_ACCESS_KEY_ID=minioadmin
      - AWS_SECRET_ACCESS_KEY=minioadmin
      - HIVE_SERVER2_THRIFT_BIND_HOST=spark-thrift
    networks:
      - dbt_sample_network

  postgresdb:
    image: 'postgres:latest'
    hostname: postgresdb
    container_name: postgresdb
    ports:
      - '5432:5432'
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      POSTGRES_DB: metastore_db
    volumes:
      - ./.hive-metastore:/var/lib/postgresql/data
    networks:
      - dbt_sample_network

  minio:
    image: 'minio/minio:latest'
    hostname: minio
    container_name: minio
    ports:
      - '9000:9000'
      - '9001:9001'
    volumes:
      - .minio-data:/data
    environment:
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
      MINIO_API_SELECT_PARQUET: "on"
    command: server --console-address ":9001" /data
    networks:
      - dbt_sample_network

  createbuckets:
    image: minio/mc
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c "
      /usr/bin/mc config host add myminio http://minio:9000 minioadmin minioadmin;
      /usr/bin/mc mb myminio/warehouse;
      /usr/bin/mc policy download myminio/warehouse;
      exit 0;
      "
    networks:
      - dbt_sample_network

networks:
  dbt_sample_network:
    driver: bridge