Merge pull request #147 from ImperialCollegeLondon/drunc-image-profiles

Use service profiles to specify drunc-lite use case

.dockerignore

@@ -6,3 +6,5 @@
 !drunc_ui
 !db
 db/*
+!drunc_docker_service_lite/requirements.txt
+!drunc_docker_service/process-manager-kafka.json

README.md

@@ -11,50 +11,83 @@ This repo defines the web interface for various drunc tools. Including:
 - The Processes Manager
 - The Controller
 
-## Running the App
+## Docker Setup
 
-_Note that this depends on a very large docker base image._
+This repository provides a Docker Compose configuration providing the requisite services
+suitable for development of the interfaces using realistic test data. Working with the
+full docker stack requires building a very larger docker base image and has heavy
+resource requirements when running. For this reason a "lite" stack is also provided that
+can be used for development of the Process Manager UI.
 
-To run with a demo version of the drunc process manager, run it with docker compose:
+The full and lite stack are defined using [Docker Compose profiles]. It is recommended
+to set the `COMPOSE_PROFILES` environment variable to select which stack you are working
+with. Use `COMPOSE_PROFILES=drunc` for the full stack or `COMPOSE_PROFILES=drunc-lite`
+for the lite stack.
+
+You can also use the `--profile` flag when invoking `docker compose` however the
+instructions in this file assume you are using the environment variable.
+
+[Docker Compose profiles]: https://docs.docker.com/compose/how-tos/profiles/
+
+## Process Manager UI
+
+You can use either the full or lite Docker Compose profiles when working with the
+Process Manager UI.
+
+### Getting Started with the Process Manager UI
+
+Check you have a value set for `COMPOSE_PROFILES` (see above). Start the docker stack
+with:
 
 ```bash
 docker compose up
 ```
 
-It can take a few moment for the services to boot but the application should then be
-available in the browser at <http://localhost:8000>. Authentication is required to work
-with the application so you need to create a user account to work with:
+It can take a few moments for the services to boot but the application should then be
+available in the browser at <http://localhost:8000>.
+
+__Note that if you have the Process Manager UI already open in a browser you may see
+backtraces from unsuccessful requests generated by the UI. These should stop once drunc
+has finished starting up.__
+
+Authentication is required to work with the application so you need to create a user
+account to work with:
 
 ```bash
 docker compose exec app python manage.py createsuperuser
 ```
 
 and follow the prompts. You should then be able to use the details you supplied to pass
 the login screen. You can use the "boot" button on the main page to create simple
-processes to experiment with. You can also do this via the command line:
+processes to experiment with.
 
-```bash
-docker compose exec app python scripts/talk_to_process_manager.py
-```
+Take the services down with `docker compose down` or by pressing Ctrl+C in the
+corresponding terminal.
+
+## Controller UI
+
+You must use the full Docker Compose profile when working with the Controller UI.
 
-To boot a more realistic test session:
+### Getting Started with the Controller UI
+
+Check you have `COMPOSE_PROFILES=drunc` (see above). As with the Process Manager UI
+start the Docker Compose services and create a user account if not already done. You can
+then boot a full test session:
 
 ```bash
-docker compose exec drunc bash -c "source env.sh && drunc-process-manager-shell grpc://localhost:10054 boot test/config/test-session.data.xml test-session"
+docker compose exec drunc /boot_test_session.sh
 ```
 
 _Note that the above consumes several Gb of memory._
 
 Once booted you can interact with the root controller via:
 
 ```bash
-docker compose exec drunc bash -c "source env.sh && drunc-controller-shell grpc://localhost:3333"
+docker compose exec drunc /entrypoint.sh drunc-controller-shell grpc://localhost:3333
 ```
 
-For details of working with the controller see the [drunc wiki].
-
-Take the services down with `docker compose down` or by pressing Ctrl+C in the
-corresponding terminal.
+For details of working with the controller see the [drunc wiki]. You should also be able
+to see the booted processes in the Process Manager UI.
 
 [drunc wiki]: https://github.com/DUNE-DAQ/drunc/wiki/Controller
 

docker-compose.yml

@@ -14,16 +14,33 @@ services:
  - db:/usr/src/app/db
  environment:
  - PROCESS_MANAGER_URL=drunc:10054
- depends_on:
- - drunc
  drunc:
  build: ./drunc_docker_service/
+ profiles:
+ - drunc
  command:
  - bash
  - -c
  - |
  /usr/sbin/sshd &&
- source env.sh &&
+ drunc-process-manager --log-level debug file:///process-manager-kafka.json
+ expose:
+ - 10054
+ depends_on:
+ kafka:
+ condition: service_healthy
+ drunc-lite:
+ build:
+ context: ./drunc_docker_service/
+ dockerfile: ./Dockerfile_lite
+ container_name: drunc
+ profiles:
+ - drunc-lite
+ command:
+ - bash
+ - -c
+ - |
+ service ssh start &&
  drunc-process-manager --log-level debug file:///process-manager-kafka.json
  expose:
  - 10054

drunc_docker_service/Dockerfile

@@ -7,7 +7,9 @@ SHELL ["/bin/bash", "-c"]
 
 WORKDIR /basedir
 
-# for frozen release
+# setup for the the dune development environment that provides the full set of
+# dependencies required for booting of the controller test session
+# See https://github.com/DUNE-DAQ/drunc/wiki/Setup-drunc-with-DUNE-DAQ for explanation
 RUN source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh && \
  setup_dbt latest_v5 && \
  dbt-create fddaq-v5.1.0-a9 fddaq-v5.1.0-a9 && \
@@ -18,18 +20,18 @@ RUN source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh && \
  pip install git+https://github.com/DUNE-DAQ/[email protected]
 WORKDIR /basedir/fddaq-v5.1.0-a9/
 
-COPY process-manager-no-kafka.json process-manager-kafka.json data/
+COPY process-manager-kafka.json /
 
-RUN mkdir -p /root/.ssh && \
- ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" && \
- cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-
-RUN echo "PermitRootLogin without-password" >> /etc/ssh/sshd_config && \
- echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config && \
- echo "UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
- ssh-keygen -A
+# setup ssh server config and keys
+ADD ssh-setup.sh /
+RUN bash /ssh-setup.sh
 
 RUN mkdir -p /usr/src/app
-ENTRYPOINT []
+ADD entrypoint.sh /
+RUN chmod a+x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
 
 ADD wibeth_output_all_zeros.bin /cvmfs/dunedaq.opensciencegrid.org/assets/files/d/d/1/wibeth_output_all_zeros.bin
+
+ADD boot_test_session.sh /
+RUN chmod a+x /boot_test_session.sh

drunc_docker_service/Dockerfile_lite

@@ -0,0 +1,16 @@
+FROM python:3.11-slim-bookworm
+
+RUN apt-get update \
+ && apt-get -y install --no-install-recommends git ssh openssh-server \
+ && rm -fr /var/lib/apt/lists/*
+
+COPY lite_requirements.txt /requirements.txt
+RUN pip install --no-cache-dir -r /requirements.txt
+
+COPY process-manager-kafka.json /
+
+# setup ssh server config and keys
+ADD ssh-setup.sh /
+RUN bash /ssh-setup.sh
+
+RUN mkdir -p /usr/src/app

drunc_docker_service/boot_test_session.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# setup dunedaq environment
+. /basedir/fddaq-v5.1.0-a9/env.sh
+
+# boot full test session
+drunc-process-manager-shell grpc://localhost:10054 boot test/config/test-session.data.xml test-session

drunc_docker_service/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# setup dunedaq environment
+. /basedir/fddaq-v5.1.0-a9/env.sh
+
+exec "$@"

drunc_docker_service/lite_requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/DUNE-DAQ/[email protected]
+git+https://github.com/DUNE-DAQ/[email protected]

drunc_docker_service/ssh-setup.sh

@@ -0,0 +1,16 @@
+# Setup the ssh server configuration and keys
+
+set -e
+
+# create an ssh key for the root user
+mkdir -p /root/.ssh
+ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""
+
+# allow the key to be used to authenticate ssh connections
+cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# configure the ssh server to allow the root user to login only with an ssh key
+echo "PermitRootLogin without-password" >> /etc/ssh/sshd_config
+
+# generate a unique host key
+ssh-keygen -A