diff --git a/README.md b/README.md index 1be096c2e..70450722f 100644 --- a/README.md +++ b/README.md @@ -135,23 +135,25 @@ To use the Docker container as an interactive virtual environment, you can run a To run a submission end-to-end in a containerized environment see [Getting Started Document](./getting_started.md#run-your-submission-in-a-docker-container). ### Using Singularity/Apptainer instead of Docker -Since many compute clusters don't allow the usage of Docker due to securtiy concerns and instead encourage the use of [Singularity/Apptainer](https://github.com/apptainer/apptainer) (formerly Singularity, now called Apptainer), we also provide instructions on how to build an Apptainer container based on the here provided Dockerfile. - -To convert the Dockerfile into an Apptainer definition file, we will use [spython](https://github.com/singularityhub/singularity-cli): +Since many compute clusters don't allow the usage of Docker due to securtiy concerns and instead encourage the use of [Singularity/Apptainer](https://github.com/apptainer/apptainer) (formerly Singularity, now called Apptainer), we also provide an Apptainer recipe that can be used to build an image by running ```bash -pip3 install spython -cd algorithmic-efficiency/docker -spython recipe Dockerfile &> Singularity.def +singularity build --fakeroot .sif Singularity.def ``` -Now we can build the Apptainer image by running +Then, to start a shell session with GPU support (by using the `--nv` flag), we can run ```bash -singularity build --fakeroot .sif Singularity.def +singularity shell --bind $HOME/data:/data,$HOME/experiment_runs:/experiment_runs \ + --nv .sif ``` -To start a shell session with GPU support (by using the `--nv` flag), we can run + +Note the `--bind` flag which, similarly to Docker, allows to bind specific paths on the host system and the container, as explained [here](https://docs.sylabs.io/guides/3.7/user-guide/bind_paths_and_mounts.html). + +Also note that `Singularity.def` was automatically generated from the `Dockerfile` using [spython](https://github.com/singularityhub/singularity-cli), as follows: ```bash -singularity shell --nv .sif +pip3 install spython +cd algorithmic-efficiency/docker +python scripts/singularity_converter.py -i Dockerfile -o Singularity.def ``` -Similarly to Docker, Apptainer allows you to bind specific paths on the host system and the container by specifying the `--bind` flag, as explained [here](https://docs.sylabs.io/guides/3.7/user-guide/bind_paths_and_mounts.html). +Users that wish to customize their images are invited to check the respective files. # Getting Started For instructions on developing and scoring your own algorithm in the benchmark see [Getting Started Document](./getting_started.md). diff --git a/docker/Singularity.def b/docker/Singularity.def new file mode 100644 index 000000000..5f5c31d60 --- /dev/null +++ b/docker/Singularity.def @@ -0,0 +1,76 @@ +Bootstrap: docker +From: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 +Stage: spython-base + +%post +# Dockerfile for AlgoPerf environment. +# To build Docker image with only Jax GPU installed: +# docker build -t --build-arg framework=jax +# To build Docker image with Pytorch GPU installed: +# docker build -t --build-arg framework=pytorch + +# To build Docker image + +# Installing machine packages +echo "Setting up machine" +apt-get update +apt-get install -y curl tar +DEBIAN_FRONTEND=noninteractive apt-get install -y git python3 pip wget ffmpeg +apt-get install libtcmalloc-minimal4 +apt-get install unzip +apt-get install pigz +export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4 + +# Install GCP tools +echo "Setting up gsutil" +curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-413.0.0-linux-x86_64.tar.gz +tar -xf google-cloud-cli-413.0.0-linux-x86_64.tar.gz +yes | ./google-cloud-sdk/install.sh + +# Directory setup for input and output +echo "Setting up directories for data and experiment_runs" +mkdir -p data/ +mkdir -p experiment_runs/ + +# Install Algorithmic efficiency repo +echo "Setting up algorithmic_efficiency repo" +branch="main" +framework="both" +git_url=https://github.com/mlcommons/algorithmic-efficiency.git +git clone $git_url && cd /algorithmic-efficiency +cd /algorithmic-efficiency && git checkout $branch + +cd /algorithmic-efficiency && pip install -e '.[full]' + +if [ "$framework" = "jax" ] ; then \ +echo "Installing Jax GPU" \ +&& cd /algorithmic-efficiency \ +&& pip install -e '.[jax_gpu]' -f 'https://storage.googleapis.com/jax-releases/jax_cuda_releases.html' \ +&& pip install -e '.[pytorch_cpu]' -f 'https://download.pytorch.org/whl/torch_stable.html'; \ +elif [ "$framework" = "pytorch" ] ; then \ +echo "Installing Pytorch GPU" \ +&& cd /algorithmic-efficiency \ +&& pip install -e '.[jax_cpu]' \ +&& pip install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/torch_stable.html'; \ +elif [ "$framework" = "both" ] ; then \ +echo "Installing Jax GPU and Pytorch GPU" \ +&& cd /algorithmic-efficiency \ +&& pip install -e '.[jax_gpu]' -f 'https://storage.googleapis.com/jax-releases/jax_cuda_releases.html' \ +&& pip install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/torch_stable.html'; \ +else \ +echo "Invalid build-arg $framework: framework should be either jax, pytorch or both." >&2 \ +&& exit 1 ; \ +fi + +cd /algorithmic-efficiency && pip install -e '.[wandb]' + +cd /algorithmic-efficiency && git fetch origin +cd /algorithmic-efficiency && git pull + +# Todo: remove this, this is temporary for developing +chmod a+x /algorithmic-efficiency/docker/scripts/startup.sh + +%runscript +exec bash /algorithmic-efficiency/docker/scripts/startup.sh "$@" +%startscript +exec bash /algorithmic-efficiency/docker/scripts/startup.sh "$@" \ No newline at end of file diff --git a/docker/scripts/singularity_converter.py b/docker/scripts/singularity_converter.py new file mode 100644 index 000000000..5834d10db --- /dev/null +++ b/docker/scripts/singularity_converter.py @@ -0,0 +1,45 @@ +""" +This script is a modification of the +``spython recipe Dockerfile &> Singularity.def`` command, implemented here: +github.com/singularityhub/singularity-cli/blob/master/spython/client/recipe.py + +It converts the Docker recipy to Singularity, but suppressing any %files +command. Usage example: + +python singularity_converter.py -i Dockerfile -o Singularity.def +""" + + +import argparse +# +import spython +from spython.main.parse.parsers import get_parser +from spython.main.parse.writers import get_writer + +# globals +ENTRY_POINT = "/bin/bash" # seems to be a good default +FORCE = False # seems to be a good default +# +parser = argparse.ArgumentParser(description="Custom Singularity converter") +parser.add_argument('-i', '--input', type=str, + help="Docker input path", default="Dockerfile") +parser.add_argument('-o', '--output', type=str, + help="Singularity output path", default="Singularity.def") +args = parser.parse_args() +INPUT_DOCKERFILE_PATH = args.input +OUTPUT_SINGULARITY_PATH = args.output + +# create Docker parser and Singularity writer +parser = get_parser("docker") +writer = get_writer("singularity") + +# parse Dockerfile into Singularity and suppress %files commands +recipeParser = parser(INPUT_DOCKERFILE_PATH) +recipeWriter = writer(recipeParser.recipe) +key, = recipeParser.recipe.keys() +recipeWriter.recipe[key].files = [] + +# convert to string and save to output file +result = recipeWriter.convert(runscript=ENTRY_POINT, force=FORCE) +with open(OUTPUT_SINGULARITY_PATH, "w") as f: + f.write(result)