docker-compose.yml

version: '3.6'

volumes:
    prometheus_data: {}
    grafana_data: {}
    zookeeper-data:
      driver: local
    zookeeper-log:
      driver: local
    kafka-data:
      driver: local

services:

  # ZooKeeper is a centralized service for maintaining configuration information,
  # naming, providing distributed synchronization, and providing group services.
  # It provides distributed coordination for our Kafka cluster.
  # http://zookeeper.apache.org/
  zookeeper:
    image: confluentinc/cp-zookeeper
    container_name: zookeeper
    # ZooKeeper is designed to "fail-fast", so it is important to allow it to
    # restart automatically.
    restart: unless-stopped
    volumes:
      - zookeeper-data:/var/lib/zookeeper/data
      - zookeeper-log:/var/lib/zookeeper/log
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181


  # Kafka is a distributed streaming platform. It is used to build real-time streaming
  # data pipelines that reliably move data between systems and platforms, and to build
  # real-time streaming applications that transform or react to the streams of data.
  # http://kafka.apache.org/
  kafka:
    image: confluentinc/cp-kafka
    container_name: kafka
    volumes:
      - kafka-data:/var/lib/kafka
    restart: unless-stopped
    environment:
      # Required. Instructs Kafka how to get in touch with ZooKeeper.
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_NUM_PARTITIONS: 1
      KAFKA_COMPRESSION_TYPE: gzip
      # Required when running in a single-node cluster, as we are. We would be able to take the default if we had
      # three or more nodes in the cluster.
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      # Required. Kafka will publish this address to ZooKeeper so clients know
      # how to get in touch with Kafka. "PLAINTEXT" indicates that no authentication
      # mechanism will be used.
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
    links:
      - zookeeper


  # The Kafka REST Proxy provides a RESTful interface to a Kafka cluster.
  # It makes it easy to produce and consume messages, view the state
  # of the cluster, and perform administrative actions without using
  # the native Kafka protocol or clients.
  # https://github.com/confluentinc/kafka-rest
  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:latest
    container_name: kafka-rest-proxy
    environment:
      # Specifies the ZooKeeper connection string. This service connects
      # to ZooKeeper so that it can broadcast its endpoints as well as
      # react to the dynamic topology of the Kafka cluster.
      KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181
      # The address on which Kafka REST will listen for API requests.
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      # Required. This is the hostname used to generate absolute URLs in responses.
      # It defaults to the Java canonical hostname for the container, which might
      # not be resolvable in a Docker environment.
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      # The list of Kafka brokers to connect to. This is only used for bootstrapping,
      # the addresses provided here are used to initially connect to the cluster,
      # after which the cluster will dynamically change. Thanks, ZooKeeper!
      KAFKA_REST_BOOTSTRAP_SERVERS: kafka:9092
    # Kafka REST relies upon Kafka, ZooKeeper
    # This will instruct docker to wait until those services are up
    # before attempting to start Kafka REST.
    restart: unless-stopped
    ports:
      - "9999:8082"
    depends_on:
      - zookeeper
      - kafka

  # # Browse Kafka topics and understand what's happening on your cluster.
  # # Find topics / view topic metadata / browse topic data
  # # (kafka messages) / view topic configuration / download data.
  # # https://github.com/Landoop/kafka-topics-ui
  # kafka-topics-ui:
  #   image: landoop/kafka-topics-ui:latest
  #   container_name: kafka-topics-ui
  #   platform: linux/amd64
  #   ports:
  #     - "8081:8000"
  #   environment:
  #     # Required. Instructs the UI where it can find the Kafka REST Proxy.
  #     KAFKA_REST_PROXY_URL: "http://kafka-rest-proxy:8082/"
  #     # This instructs the docker image to use Caddy to proxy traffic to kafka-topics-ui.
  #     PROXY: "true"
  #   # kafka-topics-ui relies upon Kafka REST.
  #   # This will instruct docker to wait until those services are up
  #   # before attempting to start kafka-topics-ui.
  #   depends_on:
  #     - kafka-rest-proxy

  # Kafka Connect, an open source component of Apache Kafka,
  # is a framework for connecting Kafka with external systems
  # such as databases, key-value stores, search indexes, and file systems.
  # https://docs.confluent.io/current/connect/index.html
  kafka-connect:
    image: confluentinc/cp-kafka-connect:latest
    hostname: kafka-connect
    container_name: kafka-connect
    environment:
      # Required.
      # The list of Kafka brokers to connect to. This is only used for bootstrapping,
      # the addresses provided here are used to initially connect to the cluster,
      # after which the cluster can dynamically change. Thanks, ZooKeeper!
      CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
      # Required. A unique string that identifies the Connect cluster group this worker belongs to.
      CONNECT_GROUP_ID: kafka-connect-group
      # Connect will actually use Kafka topics as a datastore for configuration and other data. #meta
      # Required. The name of the topic where connector and task configuration data are stored.
      CONNECT_CONFIG_STORAGE_TOPIC: kafka-connect-meta-configs
      # Required. The name of the topic where connector and task configuration offsets are stored.
      CONNECT_OFFSET_STORAGE_TOPIC: kafka-connect-meta-offsets
      # Required. The name of the topic where connector and task configuration status updates are stored.
      CONNECT_STATUS_STORAGE_TOPIC: kafka-connect-meta-status
      # Required. Converter class for key Connect data. This controls the format of the
      # data that will be written to Kafka for source connectors or read from Kafka for sink connectors.
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      # Required. Converter class for value Connect data. This controls the format of the
      # data that will be written to Kafka for source connectors or read from Kafka for sink connectors.
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      # Required. The hostname that will be given out to other workers to connect to.
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_REST_PORT: 8083
      # The next three are required when running in a single-node cluster, as we are.
      # We would be able to take the default (of 3) if we had three or more nodes in the cluster.
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      #Connectos path
      CONNECT_PLUGIN_PATH: "/usr/share/java,/data/connectors/"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
    restart: unless-stopped
    volumes:
      - ./kafka_connect/data:/data
    command: 
      - bash 
      - -c 
      - |
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run & 
        #
        echo "Waiting for Kafka Connect to start listening on http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors ⏳"
        while [ $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) -ne 200 ] ; do 
          echo -e $$(date) " Kafka Connect listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) " (waiting for 200)"
          sleep 5 
        done
        nc -vz $$CONNECT_REST_ADVERTISED_HOST_NAME $$CONNECT_REST_PORT
        echo -e "\n--\n+> Creating Kafka Connect MongoDB sink Current PATH ($$PWD)"
        /data/scripts/create_mongo_sink.sh 
        echo -e "\n--\n+> Creating MQTT Source Connect Current PATH ($$PWD)"
        /data/scripts/create_mqtt_source.sh
        echo -e "\n--\n+> Creating Kafka Connect Prometheus sink Current PATH ($$PWD)"
        /data/scripts/create_prometheus_sink.sh
        sleep infinity
    # kafka-connect relies upon Kafka and ZooKeeper.
    # This will instruct docker to wait until those services are up
    # before attempting to start kafka-connect.
    depends_on:
      - zookeeper
      - kafka

  # # This is a web tool for Kafka Connect for setting up and managing connectors for multiple connect clusters.
  # # https://github.com/Landoop/kafka-connect-ui
  # kafka-connect-ui:
  #   image: landoop/kafka-connect-ui:latest
  #   container_name: kafka-connect-ui
  #   # kafka-connect-ui binds to port 8000, but we are going to expose it on our local
  #   # machine on port 8002.
  #   ports:
  #     - "8082:8000"
  #   environment:
  #     # Required. Instructs the UI where it can find Kafka Connect.
  #     CONNECT_URL: "http://kafka-connect:8083/"
  #     # This instructs the docker image to use Caddy to proxy traffic to kafka-connect-ui.
  #     PROXY: "true"
  #   # kafka-connect-ui relies upon Kafka Connect.
  #   # This will instruct docker to wait until those services are up
  #   # before attempting to start kafka-connect-ui.
  #   depends_on:
  #     - kafka-connect

  # Web client for ZooNavigator, web-based browser & editor for ZooKeeper.
  # https://github.com/elkozmon/zoonavigator
  zoonavigator:
    image: elkozmon/zoonavigator:latest
    container_name: zoonavigator
    restart: unless-stopped
    ports:
      - "9000:9000"
    environment:
      # The port on which the api service will listen for incoming connections.
      HTTP_PORT: 9000
    depends_on:
      - zookeeper

  # Eclipse Mosquitto is an open source (EPL/EDL licensed) message broker that implements the MQTT protocol versions 5.0, 3.1.1 and 3.1. Mosquitto is lightweight and is suitable for use on all devices from low power single board computers to full servers.
  # The MQTT protocol provides a lightweight method of carrying out messaging using a publish/subscribe model. This makes it suitable for Internet of Things messaging such as with low power sensors or mobile devices such as phones, embedded computers or microcontrollers.
  # The Mosquitto project also provides a C library for implementing MQTT clients, and the very popular mosquitto_pub and mosquitto_sub command line MQTT clients.
  mosquitto:
    image: eclipse-mosquitto:latest
    hostname: mosquitto
    container_name: mosquitto
    restart: unless-stopped
    ports:
      - "1883:1883"
      - "9001:9001"
    volumes:
      - ./mosquitto/config:/mosquitto/config

  mongo:
    image: mongo
    container_name: mongo
    env_file:
      - .env
    restart: unless-stopped
    environment:
      - MONGO_INITDB_ROOT_USERNAME=${MONGO_ROOT_USER}
      - MONGO_INITDB_ROOT_PASSWORD=${MONGO_ROOT_PASSWORD}
      - MONGO_INITDB_DATABASE=${MONGO_DB}

  # Web-based MongoDB admin interface, written with Node.js and express
  mongo-express:
    image: mongo-express
    container_name: mongo-express
    env_file:
      - .env
    restart: unless-stopped
    environment:
      - ME_CONFIG_MONGODB_SERVER=mongo
      - ME_CONFIG_MONGODB_PORT=27017
      - ME_CONFIG_MONGODB_ENABLE_ADMIN=true
      - ME_CONFIG_MONGODB_AUTH_DATABASE=admin
      - ME_CONFIG_MONGODB_ADMINUSERNAME=${MONGO_ROOT_USER}
      - ME_CONFIG_MONGODB_ADMINPASSWORD=${MONGO_ROOT_PASSWORD}
      - ME_CONFIG_BASICAUTH_USERNAME=${MONGOEXPRESS_LOGIN}
      - ME_CONFIG_BASICAUTH_PASSWORD=${MONGOEXPRESS_PASSWORD}
    depends_on:
      - mongo
    ports:
      - "8084:8081"
      
  
  # Grafana is a multi-platform open source analytics and interactive visualization web application. 
  # It provides charts, graphs, and alerts for the web when connected to supported data sources. 
  # It is expandable through a plug-in system. End users can create complex monitoring dashboards using interactive query builders.
  # https://github.com/hanattaw/grafana-flowcharting
  # from link above, some fixed for plugin flowcharting. You need to manually install plugin.
  grafana:
    image: grafana/grafana:latest-ubuntu
    container_name: grafana
    user: '0'
    volumes:
      - ./grafana/data:/var/lib/grafana
      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
      - ./grafana/datasources:/etc/grafana/provisioning/datasources
    environment:
      - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-worldmap-panel,grafana-piechart-panel
      - GF_USERS_ALLOW_SIGN_UP=false
    restart: unless-stopped
    links:
       - prometheus
    ports:
      - '8085:3000'
  
  # Prometheus is a free software application used for event monitoring and alerting.
  # It records real-time metrics in a time series database built using a HTTP pull model, with flexible queries and real-time alerting.
  # The project is written in Go and licensed under the Apache 2 License, with source code available on GitHub,
  # and is a graduated project of the Cloud Native Computing Foundation, along with Kubernetes and Envoy.
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - ./prometheus/:/etc/prometheus/
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'
    restart: unless-stopped
    ports:
      - '8086:9090'
    depends_on:
      - nodeexporter
      
  # Exporter for machine metrics    
  nodeexporter:
    image: prom/node-exporter:v0.18.1
    container_name: nodeexporter
    hostname: nodeexporter
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
    restart: unless-stopped
    ports:
      - '9100:9100'

  # Kafka exporter for Prometheus
  kafka-exporter:
    image: bitnami/kafka-exporter:latest
    container_name: kafka-exporter
    hostname: kafka-exporter
    command:
      - '--kafka.server=kafka:9092'
      - '--web.listen-address=kafka-exporter:9308'
      - '--web.telemetry-path=/metrics'
      - '--log.level=debug'
    restart: unless-stopped

 
  # # IoT Sensor 1
  iot_sensor_1:
    image: ssanchez11/iot_sensor:0.0.1-SNAPSHOT
    container_name: iot_sensor_1
    restart: unless-stopped
    environment:
      - sensor.id=${IOT_SENSOR_1_ID}
      - sensor.name=${IOT_SENSOR_1_NAME}
      - sensor.place.id=${IOT_SENSOR_1_PLACE_ID}

  # # # # IoT Sensor 2
  # iot_sensor_2:
  #   image: ssanchez11/iot_sensor:0.0.1-SNAPSHOT
  #   container_name: iot_sensor_2
  #   restart: unless-stopped
  #   environment:
  #     - sensor.id=${IOT_SENSOR_2_ID}
  #     - sensor.name=${IOT_SENSOR_2_NAME}
  #     - sensor.place.id=${IOT_SENSOR_2_PLACE_ID}
  # # # # IoT Sensor 1

  # iot_sensor_3:
  #   image: ssanchez11/iot_sensor:0.0.1-SNAPSHOT
  #   container_name: iot_sensor_3
  #   restart: unless-stopped
  #   environment:
  #     - sensor.id=${IOT_SENSOR_3_ID}
  #     - sensor.name=${IOT_SENSOR_3_NAME}
  #     - sensor.place.id=${IOT_SENSOR_3_PLACE_ID}

  # IoT Processor
  iot-processor:
    image: ssanchez11/iot_processor:0.0.1-SNAPSHOT
    container_name: iot-processor
    restart: unless-stopped
    ports:
      - '8080:8080'
      
  pushgateway:
    image: prom/pushgateway:v0.8.0
    container_name: pushgateway
    restart: unless-stopped
    ports:
      - '9091:9091'

  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:latest
    ports:
      - 18080:8080
    depends_on:
      - kafka
    environment:
      DYNAMIC_CONFIG_ENABLED: 'true'
      KAFKA_CLUSTERS_0_NAME: wizard_test
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092