docker-compose.yml

version: "3.8" 

services:
  # Extractor service
  hf_extractor_with_gpu:
    profiles: [gpu]
    container_name: hf_gpu
    build: 
      context: .
      dockerfile: ./extractors/hf_extractor/Dockerfile.gpu  # Path to extractor Dockerfile
    # Define volumes to mount extractor scripts or dependencies
    volumes:
      - ./extractors/hf_extractor:/app
      - ../data/transform_queue:/transform_queue
      - ../data/datasets:/datasets
      - ./config_data:/config_data
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            capabilities: [gpu]
    dns:
      - 1.1.1.1
    # depends_on:
    #   - mysql
    # Uncomment this line to run the script
    # command: ["python3", "ExtractFromDataset.py"]
    # Uncomment this line to develop on the container
    command: ["tail", "-f", "/dev/null"]
    networks:
      - mlentory_network
  
  hf_extractor_with_no_gpu:
    profiles: [no_gpu]
    container_name: hf_no_gpu
    build: 
      context: .
      dockerfile: ./extractors/hf_extractor/Dockerfile.no_gpu  # Path to extractor Dockerfile
    # Define volumes to mount extractor scripts or dependencies
    volumes:
      - ./extractors/hf_extractor:/app
      - ../data/transform_queue:/transform_queue
      - ../data/datasets:/datasets
      - ./config_data:/config_data
    dns:
      - 1.1.1.1
    # Uncomment this line to test the script
    # command: ["python3", "main.py"]
    # Uncomment this line to develop on the container
    command: ["tail", "-f", "/dev/null"]
    networks:
      - mlentory_network

  ### Module in charge of transforming the data to the RDA FAIR4ML schema
  transform:
    profiles:
      - gpu
      - no_gpu
    container_name: transform
    build: 
      context: .
      dockerfile: ./transform/Dockerfile  # Path to extractor Dockerfile
    # Define volumes to mount extractor scripts or dependencies
    volumes:
      - ./transform:/app
      - ../data/transform_queue:/transform_queue
      - ../data/load_queue:/load_queue
      - ../data/datasets:/datasets
      - ./config_data:/config_data
    dns:
      - 1.1.1.1

    # Uncomment this line to test the script
    # command: ["tail", "-f", "/dev/null"]
    command: ["python3", "main.py"]
    networks:
      - mlentory_network
    # Uncomment for production
    #command: ["python3", "ExtractFromDataset.py"]

  ### Module in charge of loading the data into the database
  load:    
    depends_on:
      - transform
      - postgres
      - virtuoso
      - elastic
    profiles:
      - gpu
      - no_gpu
    container_name: load
    build: 
      context: .
      dockerfile: ./load/Dockerfile  # Path to extractor Dockerfile
    # Define volumes to mount extractor scripts or dependencies
    volumes:
      - ./load:/app
      - ../data/load_queue:/load_queue
      - ../data/virtuoso_data/kg_files:/kg_files
      - ./config_data:/config_data
      # Necessary to run docker commands inside other containers
      - /var/run/docker.sock:/var/run/docker.sock 
    dns:
      - 1.1.1.1

    # Uncomment this line to test the script
    # command: ["tail", "-f", "/dev/null"]
    command: ["python3", "main.py"]
    networks:
      - mlentory_network
  
  ### SQL database
  postgres:
    profiles:
        - gpu
        - no_gpu
    image: postgres:14.1-alpine
    container_name: postgres
    environment:
      POSTGRES_USER: user
      POSTGRES_PASSWORD: password
      POSTGRES_DB: history_DB
    ports:
      - '5432:5432'
    volumes:
      - ./load/sql_files/init.sql:/docker-entrypoint-initdb.d/init.sql
      - ../data/postgres_data:/var/lib/postgresql/data
    networks:
      - mlentory_network

  # anzograph:
  #   image: cambridgesemantics/anzograph:latest
  #   ports:
  #     - "8085:8080"
  #     - "8443:8443"
  #   volumes:
  #     - /DB/anzograph-data:/opt/anzograph-data

  ### RDF database
  virtuoso:
    profiles:
      - gpu
      - no_gpu
    image: openlink/virtuoso-opensource-7:latest
    hostname: virtuoso
    container_name: virtuoso
    ports:
      - "1111:1111"
      - "8890:8890"
    environment:
      DBA_PASSWORD: my_strong_password
    volumes:
      - ../data/virtuoso_data:/opt/virtuoso-opensource/database
    networks:
      - mlentory_network

  ### Component to index data based on specific paramaters in Elasticsearch
  elastic:
    profiles:
      - gpu
      - no_gpu
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
    container_name: elastic
    hostname: elastic
    # privileged: true
    environment:
      - node.name=elastic
      - "transport.host=localhost"
      - "bootstrap.system_call_filter=false"
      # - discovery.type=single-node
      - cluster.name=es-docker-cluster
      - cluster.initial_master_nodes=elastic
      - bootstrap.memory_lock=true
      # - node.max_local_storage_nodes=2
      - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ../data/elasticsearch_data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - mlentory_network

# networks:
#   my-macvlan-net:
#     external: true
#     name: my-macvlan-net
#   host:
#     external: true

networks:
  mlentory_network:
    external: true