docker-compose.yml

version: '2'
services:
  kafka:
   image: spotify/kafka
   hostname: kafka
   container_name: kafka
   domainname: hadoop
   networks:
      - hadoop
   environment:
      - ADVERTISED_HOST=192.168.1.200
      - ADVERTISED_PORT=9092
      - KAFKA=localhost:9092
      - ZOOKEEPER=localhost:2181
   ports:
      - "2181:2181"
      - "9092:9092"

  namenode:
    image: bde2020/hadoop-namenode:1.0.0
    hostname: namenode
    container_name: namenode
    domainname: hadoop
    networks:
      - hadoop
    volumes:
      - ./data/namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop.env
    ports:
      - "50070:50070"
      - "8020:8020"

  datanode1:
    image: bde2020/hadoop-datanode:1.0.0
    hostname: datanode1
    container_name: datanode1
    domainname: hadoop
    networks:
      - hadoop
    volumes:
      - ./data/datanode1:/hadoop/dfs/data
    env_file:
      - ./hadoop.env

  datanode2:
    image: bde2020/hadoop-datanode:1.0.0
    hostname: datanode2
    container_name: datanode2
    domainname: hadoop
    networks:
      - hadoop
    volumes:
      - ./data/datanode2:/hadoop/dfs/data
    env_file:
      - ./hadoop.env

  spark-master:
    image: earthquakesan/hadoop-spark-master:1.0.0
    hostname: spark-master
    container_name: spark-master
    domainname: hadoop
    volumes:
      - ./target/scala-2.10:/app
    networks:
      - hadoop
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
      - SPARK_CONF_spark_eventLog_enabled=true
      - SPARK_CONF_spark_eventLog_dir=hdfs://namenode:8020/spark-logs
      - SPARK_CONF_spark_history_fs_logDirectory=hdfs://namenode:8020/spark-logs
    env_file:
      - ./hadoop.env
    ports:
        - "8080:8080"
        - "7077:7077"

  spark-worker:
    image: earthquakesan/hadoop-spark-worker:1.0.0
    hostname: spark-worker
    domainname: hadoop
    networks:
      - hadoop
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
      - SPARK_CONF_spark_eventLog_enabled=true
      - SPARK_CONF_spark_eventLog_dir=hdfs://namenode:8020/spark-logs
      - SPARK_CONF_spark_history_fs_logDirectory=hdfs://namenode:8020/spark-logs
      - SPARK_MASTER_URL=spark://spark-master:7077
    env_file:
      - ./hadoop.env

  spark-notebook:
    image: andypetrella/spark-notebook:0.6.3-scala-2.10.5-spark-1.6.1-hadoop-2.7.1-with-hive-with-parquet
    hostname: sparknotebook
    container_name: sparknotebook
    domainname: hadoop
    networks:
      - hadoop
    environment:
      - SPARK_NOTEBOOK_MASTER=yarn-client
    env_file:
      - ./hadoop.env
    ports:
      - "9000:9000"
      - "4040:4040"

  hue:
    image: bde2020/hdfs-filebrowser:3.9
    hostname: hdfsfb
    container_name: hdfsfb
    domainname: hadoop
    networks:
      - hadoop
    environment:
      - NAMENODE_HOST=namenode
    ports:
      - "8088:8088"

  gobblin:
    image: gobblin/gobblin-standalone:ubuntu-gobblin-latest
    hostname: gobblin
    container_name: gobblin
    domainname: gobblin
    networks:
      - hadoop
    volumes:
      - ./gobblin/conf:/etc/opt/job-conf
      - ./gobblin/work-dir:/home/gobblin/work-dir
      - ./gobblin/logs:/var/log/gobblin
    depends_on:
      - "kafka"
      - "namenode"

networks:
  hadoop:
    external: true