diff --git a/docker-cmd-mac.sh b/docker-cmd-mac.sh new file mode 100755 index 0000000..5188778 --- /dev/null +++ b/docker-cmd-mac.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# Example of docker usage for RLLIB + SUMO Utlis +# +# Author: Lara CODECA +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0. + +set -e +set -u + +IMAGE_NAME="tf-gpu-sumo-$(date +%Y-%m-%d)" +IMAGE_FOLDER="docker-image-mac" +GPU=true +GPU_OPT="--gpus all" +OPTIRUN=false +OPTIRUN_OPT="" +BUILD=false +CACHE=false +RUN=false +SCREEN=false +EXEC=false +CONTAINER="" +DEVEL_DIR="" +LEARN_DIR="" +COMMAND="" +EXP="" +DETACH=false + +function print_help { + echo "Parameters:" + echo " IMAGE name \"$IMAGE_NAME\" [-n, --image-name]" + echo " IMAGE folder \"$IMAGE_FOLDER\" [-f, --image-folder]" + echo " GPU enabled ($GPU) [--no-gpu]" + echo " OPTIRUN disabled ($OPTIRUN) [--with-optirun]" + echo " BUILD: $BUILD [-b, --build], with CACHE: $CACHE [-c, --cache]" + echo " RUN: $RUN [-r, --run], with SCREEN: $SCREEN [-s, --screen]" + echo " EXEC: $EXEC [-e, --exec], CONTAINER: \"$CONTAINER\" (use docker ps for the id)" + echo " COMMAND: \"$COMMAND\" [--cmd]" + echo " EXP: \"$EXP\" [--exp]" + echo " DETACH: ($DETACH) [--detach]" + echo " DEVELOPMENT dir \"$DEVEL_DIR\" [-d, --devel]" + echo " LEARNING dir \"$LEARN_DIR\" [-l, --learn]" +} + +for arg in "$@" +do + case $arg in ## -l=*|--lib=*) DIR="${i#*=}" is the way to retrieve the parameter + -n=*|--image-name=*) + IMAGE_NAME="${arg#*=}" + ;; + -f=*|--image-folder=*) + IMAGE_FOLDER="${arg#*=}" + ;; + --no-gpu) + GPU=false + GPU_OPT="" + ;; + --with-optirun) + OPTIRUN=true + OPTIRUN_OPT="optirun" + ;; + --detach) + DETACH=true + ;; + -b|--build) + BUILD=true + ;; + -c|--cache) # it does nothing without BUILD=true + CACHE=true + ;; + -r|--run) + RUN=true + ;; + -s|--screen) # it does nothing without RUN=true + SCREEN=true + ;; + -e=*|--exec=*) # it works only with RUN=false + EXEC=true + CONTAINER="${arg#*=}" + ;; + --cmd=*) + COMMAND="${arg#*=}" + ;; + --exp=*) + EXP="${arg#*=}" + ;; + -d=*|--devel=*) + DEVEL_DIR="${arg#*=}" + ;; + -l=*|--learn=*) + LEARN_DIR="${arg#*=}" + ;; + *) + # unknown option + echo "Unknown option \"$arg\"" + print_help + exit + ;; + esac +done + +print_help + +# Tensorflow original image +# docker run -u $(id -u):$(id -g) --gpus all -it --rm tensorflow/tensorflow:latest-gpu-py3 bash + +## Building the docker image +if [[ "$BUILD" = true ]]; then + if [[ "$CACHE" = true ]]; then + echo "Building the docker container using the cache, if present." + $OPTIRUN_OPT docker build \ + --build-arg USER_ID=$(id -u ${USER}) \ + --build-arg GROUP_ID=$(id -g ${USER}) \ + -t "$IMAGE_NAME" "$IMAGE_FOLDER" + else + echo "Building the docker container ignoring the cache, even if present." + $OPTIRUN_OPT docker build \ + --build-arg USER_ID=$(id -u ${USER}) \ + --build-arg GROUP_ID=$(id -g ${USER}) \ + --no-cache -t "$IMAGE_NAME" "$IMAGE_FOLDER" + fi +fi + +if [[ "$RUN" = true ]]; then + # My docker build + MOUNT_DEVEL="" + if [[ $DEVEL_DIR ]]; then + MOUNT_DEVEL="--mount src=$DEVEL_DIR,target=/home/alice/devel,type=bind" + fi + MOUNT_LEARN="" + if [[ $LEARN_DIR ]]; then + MOUNT_LEARN="--mount src=$LEARN_DIR,target=/home/alice/learning,type=bind" + fi + CONT_NAME="" + if [[ $EXP ]]; then + CONT_NAME="--name $EXP" + fi + if [[ "$DETACH" = true ]]; then + DETACH="-d" + else + DETACH="" + fi + CURR_UID=$(id -u) + CURR_GID=$(id -g) + RUN_OPT="-u $CURR_UID:$CURR_GID --net=host --env DISPLAY=$DISPLAY \ + --volume /tmp/.X11-unix:/tmp/.X11-unix \ + --privileged $MOUNT_DEVEL $MOUNT_LEARN \ + --shm-size 256m $GPU_OPT $CONT_NAME \ + -it $DETACH --rm $IMAGE_NAME:latest" + echo "$OPTIRUN_OPT docker run $RUN_OPT $COMMAND" + + ## Running docker + if [[ "$SCREEN" = true ]]; then + echo "Running the docker in a screen session." + screen -d -m \ + $OPTIRUN_OPT docker run $RUN_OPT $COMMAND + else + $OPTIRUN_OPT docker run $RUN_OPT $COMMAND + fi +else + if [[ "$EXEC" = true ]]; then + echo "Attaching to a running docker (see container id using 'docker ps')." + $OPTIRUN_OPT docker exec -it "$CONTAINER" /bin/bash + fi +fi diff --git a/docker-image-mac/.DS_Store b/docker-image-mac/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/docker-image-mac/.DS_Store differ diff --git a/docker-image-mac/Dockerfile b/docker-image-mac/Dockerfile new file mode 100644 index 0000000..4e3e2d5 --- /dev/null +++ b/docker-image-mac/Dockerfile @@ -0,0 +1,109 @@ +FROM tensorflow/tensorflow:latest-gpu-py3 + +LABEL Author="Lara CODECA (lara.codeca@gmail.com)" +LABEL Description="Dockerised SUMO-devel with tensorflow-gpu environment." + +ARG USER_ID +ARG GROUP_ID + +# Install system dependencies. +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ + cmake \ + gdb \ + git \ + htop \ + ipython3 \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libfox-1.6-0 \ + libfox-1.6-dev \ + libgdal-dev \ + libopenmpi-dev \ + libspatialindex-dev \ + libswscale-dev \ + libtool \ + libxerces-c-dev \ + nano \ + psmisc \ + python3.6-dev \ + python3-tk \ + python3-virtualenv \ + rsync \ + screen \ + sudo \ + swig \ + tmux \ + tree \ + vim \ + x11-apps \ + zlib1g-dev && apt-get autoremove -y + +# Install Python 3 dependencies for SUMO and scripts +RUN pip install --upgrade pip +RUN python -m pip install \ + aiohttp \ + deepdiff \ + dill \ + folium \ + gputil \ + grpcio \ + lxml \ + lz4 \ + matplotlib \ + numpy \ + opencv-python \ + pandas \ + psutil \ + pyproj \ + rtree \ + setproctitle \ + shapely \ + tqdm + +# Install Python 3 dependencies for MARL +RUN python -m pip install gym +RUN python -m pip install ray +RUN python -m pip install ray[debug] +RUN python -m pip install ray[rllib] +RUN python -m pip install ray[tune] + +# Working user +RUN groupadd --gid 222 alice && \ + useradd -m -s /bin/bash -u 502 -g 222 alice && \ + echo "alice:alice" | chpasswd && adduser alice sudo +USER alice + +# Download and install SUMO +WORKDIR /home/alice +RUN git clone --depth 1 https://github.com/eclipse/sumo.git sumo +RUN mkdir -p /home/alice/sumo/build/cmake-build-release +WORKDIR /home/alice/sumo/build/cmake-build-release +RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Release /home/alice/sumo +RUN make -j$(nproc) +RUN mkdir -p /home/alice/sumo/build/cmake-build-debug +WORKDIR /home/alice/sumo/build/cmake-build-debug +RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Debug /home/alice/sumo +RUN make -j$(nproc) + +# Directory structure +RUN mkdir -p /home/alice/devel +RUN mkdir -p /home/alice/learning +RUN mkdir -p /home/alice/libraries + +# Download & install RLLIB+SUMO Utils +WORKDIR /home/alice/libraries +RUN git clone --depth 1 https://github.com/lcodeca/rllibsumoutils.git rllibsumoutils +WORKDIR /home/alice/libraries/rllibsumoutils +USER root +RUN python -m pip install -e . + +# Learning Environment +USER alice +WORKDIR /home/alice/learning +COPY --chown=alice tf-gpu-test.py /home/alice/learning/tf-gpu-test.py +COPY --chown=alice training.sh /home/alice/learning/training.sh + +USER alice +WORKDIR /home/alice/learning +CMD ["./training.sh"] diff --git a/docker-image-mac/tf-gpu-test.py b/docker-image-mac/tf-gpu-test.py new file mode 100755 index 0000000..0197f80 --- /dev/null +++ b/docker-image-mac/tf-gpu-test.py @@ -0,0 +1,3 @@ +import tensorflow as tf +from tensorflow.python.client import device_lib +print(device_lib.list_local_devices()) diff --git a/docker-image-mac/training.sh b/docker-image-mac/training.sh new file mode 100755 index 0000000..dcc4d24 --- /dev/null +++ b/docker-image-mac/training.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Example of learning script for RLLIB + SUMO Utlis +# +# Author: Lara CODECA +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0. + +set -e + +# SUMO-dev environmental vars +export SUMO_HOME="/home/alice/sumo" +export SUMO_BIN="$SUMO_HOME/bin" +export SUMO_TOOLS="$SUMO_HOME/tools" +export PATH="$SUMO_BIN:$PATH" + +echo "Testing SUMO command:" +sumo --version + +echo "Testing Tensorflow resources:" +python tf-gpu-test.py + +echo "Testing RLLIB SUMO Utils example:" +python /home/alice/libraries/rllibsumoutils/example/ppotrain.py diff --git a/docs/howto-mac.md b/docs/howto-mac.md new file mode 100644 index 0000000..28976c4 --- /dev/null +++ b/docs/howto-mac.md @@ -0,0 +1,61 @@ +# RLLIB SUMO Docker environment for MacOS + +## Step 1: Build your docker image + +Basic command to use: + +`bash docker-cmd-mac.sh --build` + +- The default tag name of this image is `tf-gpu-sumo-{today_date}`. You can also add option such as `--image-name={new_name}` to specify your own name; +- The default path of the dockerfile to be used is `docker-image-mac`. You can adapt to your setting by using option `--image-folder`; +- The default image building process does not use cache. You can enable it by adding the option `--cache`. + +## Step 2: Run a container of the built image + +`sudo bash docker-cmd-mac.sh --image-name=tf-gpu-sumo-2020-08-03 --run --no-gpu` + +This will run `/home/alice/learning/training.sh` inside the docker container, which will then run `/home/alice/libraries/rllibsumoutils/example/ppotrain.py`. + +To facilitate your development, you can add options to synchronize the code on your local system with the docker container. The mappings are as follows: + +| Option | Path (container) | Path (host) +| ----------- | ----------- | ----------- | +| [-d, --devel] | `/home/alice/devl/` | [your_path] | +| [-l, --learn] | `/home/alice/learning/` | [your_path] | + + + + +## docker-cmd-mac.sh usage + +``` bash +Default parameters: + IMAGE name "tf-gpu-sumo-{today}" [-n, --image-name] + IMAGE folder "docker-image-mac" [-f, --image-folder] + GPU enabled (true) [--no-gpu] + OPTIRUN disabled (false) [--with-optirun] + BUILD: false [-b, --build] + with CACHE: false [-c, --cache] + RUN: false [-r, --run] + with SCREEN: false [-s, --screen] + EXEC: false [-e, --exec] + CONTAINER: "" (use docker ps for the id) + COMMAND: "" [--cmd] + EXP: "" [--exp] + DETACH: (true) [--detach] + DEVELOPMENT dir "" [-d, --devel] + LEARNING dir "" [-l, --learn] +``` + +Example of use: + +- Build the image (with GPU enabled) and launch the example learning in the terminal: + `bash docker-cmd-mac.sh --build --cache --run` +- Build the image with a specific name and launch the example learning in the terminal: + `bash docker-cmd-mac.sh --image-name=rllibsumoutils-docker --build --cache --run` +- Run the already built image in a screen: + `bash docker-cmd-mac.sh --image-name=rllibsumoutils-docker --run --screen` +- Attach to an already running docker container (using docker ps to know the id): + `bash docker-cmd-mac.sh --exec=9ed3cec06e` +- The default docker entry point in the image runs the tests. It can be changed by using: + `bash docker-cmd-mac.sh --run --cmd="/bin/bash"`