Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Pixi for dependency management #4

Merged
merged 30 commits into from
Aug 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
06a6407
Use pixi for dependency management
jayqi Jul 29, 2024
178364b
Fix tensorflow
jayqi Jul 29, 2024
bef25dc
Move shared dependencies into a base feature
jayqi Jul 30, 2024
c24358d
move pixi files
klwetstone Jul 30, 2024
e897d08
remove conda-lock files
klwetstone Jul 30, 2024
f27fc7d
dockerfile scrap work
klwetstone Jul 30, 2024
c355861
update dockerfile
klwetstone Jul 30, 2024
d1c728d
use pixi docker base image
klwetstone Jul 30, 2024
bbd31dc
work updating test_lockfile
klwetstone Aug 1, 2024
b2f40fb
update make update-lockfiles for pixi
klwetstone Aug 1, 2024
76be305
separate dockerfile to update pixi.lock
klwetstone Aug 5, 2024
33e22bb
update lockfile
klwetstone Aug 6, 2024
4a83b05
update makefile
klwetstone Aug 6, 2024
ea9b46a
remove test_lockfile.py adapted for pixi
klwetstone Aug 6, 2024
c2290a5
makefile logging
klwetstone Aug 6, 2024
b5dbb58
[Into Pixi PR #4] Fix Dockerfiles (#5)
jayqi Aug 9, 2024
8b97f0f
Use a cuda base image
jayqi Aug 9, 2024
07594a4
Hardcode python executable for test
jayqi Aug 9, 2024
8188f71
Remove extra Python
jayqi Aug 9, 2024
925ac16
pass CPU_OR_GPU to entrypoint.sh
klwetstone Aug 13, 2024
5ec5e64
log cpu or gpu in entrypoint
klwetstone Aug 14, 2024
3d6e80e
use jammy for locking
klwetstone Aug 14, 2024
91ddba1
test pixi.lock update
klwetstone Aug 15, 2024
aa6c968
hacky pixi dependency fix
klwetstone Aug 15, 2024
f1adf43
revert pixi.lock
klwetstone Aug 15, 2024
6070c3e
only log in smoke tests
klwetstone Aug 15, 2024
9d8dfec
fix mounting for lockfile generation
klwetstone Aug 15, 2024
a52df64
revert makefile changes for testing
klwetstone Aug 15, 2024
ef2d0ba
fix comment
klwetstone Aug 15, 2024
bd39cee
omit trailing slash
klwetstone Aug 16, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,19 @@ jobs:
LATEST_TAG: ${{ matrix.proc }}-latest

steps:
- name: Remove unwanted software
run: |
echo "Available storage before:"
sudo df -h
echo
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
echo "Available storage after:"
sudo df -h
echo

- uses: actions/checkout@v4

- name: Build Image
Expand All @@ -56,7 +69,9 @@ jobs:

- name: Tests packages in container
run: |
docker run $LOGIN_SERVER/$IMAGE:$SHA_TAG python -m pytest tests
docker run --network none \
$LOGIN_SERVER/$IMAGE:$SHA_TAG \
/code_execution/.pixi/envs/${{ matrix.proc }}/bin/python -m pytest tests

- name: Log into Azure
if: ${{ fromJson(env.SHOULD_PUBLISH) }}
Expand All @@ -78,4 +93,3 @@ jobs:
if: ${{ fromJson(env.SHOULD_PUBLISH) }}
run: |
docker push $LOGIN_SERVER/$IMAGE --all-tags

62 changes: 29 additions & 33 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ else
CPU_OR_GPU ?= gpu
endif

BLOCK_INTERNET ?= true

TAG := ${CPU_OR_GPU}-latest
LOCAL_TAG := ${CPU_OR_GPU}-local

Expand Down Expand Up @@ -109,61 +111,55 @@ endif
#################################################################################
# Commands for building the container if you are changing the requirements #
#################################################################################
.PHONY: build clean interact-container pack-example pack-submission pull test-container test-submission update-lockfiles
.PHONY: build clean interact-container pack-example pack-submission pull test-container test-submission update-lockfile

## Builds the container locally
build:
docker build runtime \
--build-arg CPU_OR_GPU=${CPU_OR_GPU} \
--tag ${LOCAL_IMAGE}:${LOCAL_TAG}

runtime/pixi.lock:
ifeq (,$(wildcard ./runtime/pixi.lock))
@echo Generating runtime/pixi.lock from scratch
@echo "version: 5" > runtime/pixi.lock
@echo "environments:" >> runtime/pixi.lock
@echo "packages:" >> runtime/pixi.lock
else
@echo Updating existing runtime/pixi.lock
endif

## Updates runtime environment lockfile using Docker
update-lockfile: runtime/pixi.lock
@echo Building Docker image to generate lockfile
docker build runtime \
--file runtime/Dockerfile-lock \
--tag pixi-lock:local
@echo Running lock container
docker run \
--mount type=bind,source="$(shell pwd)"/runtime/pixi.toml,target=/tmp/pixi.toml \
--mount type=bind,source="$(shell pwd)"/runtime/pixi.lock,target=/tmp/pixi.lock \
--rm \
pixi-lock:local

runtime/conda-lock-cpu.yml: runtime/environment-cpu.yml
@echo Locking the CPU environment
conda-lock \
--mamba \
-p linux-64 \
--without-cuda \
-f runtime/environment-cpu.yml \
--lockfile runtime/conda-lock-cpu.yml
conda-lock render \
-p linux-64 \
--filename-template runtime/environment-cpu.lock \
runtime/conda-lock-cpu.yml

runtime/conda-lock-gpu.yml: runtime/environment-gpu.yml
@echo Locking the GPU environment
conda-lock \
--mamba \
-p linux-64 \
--with-cuda 11.8 \
-f runtime/environment-gpu.yml \
--lockfile runtime/conda-lock-gpu.yml
conda-lock render \
-p linux-64 \
--filename-template runtime/environment-gpu.lock \
runtime/conda-lock-gpu.yml

## Updates runtime environment lockfiles
update-lockfiles: runtime/conda-lock-cpu.yml runtime/conda-lock-gpu.yml
@python runtime/tests/test_lockfile.py runtime/conda-lock-cpu.yml
@python runtime/tests/test_lockfile.py runtime/conda-lock-gpu.yml


## Ensures that your locally built image can import all the Python packages successfully when it runs
test-container: _check_image _echo_image _submission_write_perms
docker run \
${GPU_ARGS} \
${NETWORK_ARGS} \
${TTY_ARGS} \
--mount type=bind,source="$(shell pwd)"/runtime/tests,target=/tests,readonly \
--pid host \
${SUBMISSION_IMAGE_ID} \
python -m pytest -v tests
pixi run -e ${CPU_OR_GPU} python -m pytest tests


## Open an interactive bash shell within the running container (with network access)
interact-container: _check_image _echo_image _submission_write_perms
docker run \
${GPU_ARGS} \
${NETWORK_ARGS} \
--mount type=bind,source=${shell pwd}/data,target=/code_execution/data,readonly \
--mount type=bind,source="$(shell pwd)/submission",target=/code_execution/submission \
--shm-size 8g \
Expand Down
35 changes: 25 additions & 10 deletions runtime/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 mambaorg/micromamba:1.5.3-bookworm-slim
FROM ghcr.io/prefix-dev/pixi:0.26.1-jammy-cuda-11.8.0

USER root

Expand All @@ -10,24 +10,39 @@ ENV DEBIAN_FRONTEND=noninteractive \
PYTHONUNBUFFERED=1 \
SHELL=/bin/bash

# Create user andset permissions
ENV RUNTIME_USER=runtimeuser
ENV RUNTIME_UID=1000
ENV RUNTIME_GID=1000
ENV CPU_OR_GPU=$CPU_OR_GPU

RUN echo "Creating ${RUNTIME_USER} user..." \
&& groupadd --gid ${RUNTIME_GID} ${RUNTIME_USER} \
&& useradd --create-home --gid ${RUNTIME_GID} --no-log-init --uid ${RUNTIME_UID} ${RUNTIME_USER}

COPY apt.txt apt.txt
RUN apt-get update --fix-missing \
&& apt-get install -y apt-utils 2> /dev/null \
&& xargs -a apt.txt apt-get install -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /apt.txt

COPY --chown=$MAMBA_USER:$MAMBA_USER conda-lock-${CPU_OR_GPU}.yml /tmp/conda-lock.yml
RUN micromamba install --name base --yes --file /tmp/conda-lock.yml && \
micromamba clean --all --force-pkgs-dirs --yes

# Set up code execution working directory
RUN mkdir /code_execution
RUN chown -R ${MAMBA_USER}:${MAMBA_USER} /code_execution
RUN chown -R ${RUNTIME_USER}:${RUNTIME_USER} /code_execution
WORKDIR /code_execution

COPY tests /code_execution/tests
COPY entrypoint.sh /entrypoint.sh
# Switch to runtime user
USER ${RUNTIME_USER}

WORKDIR /code_execution
USER ${MAMBA_USER}
COPY pixi.lock ./pixi.lock
COPY pixi.toml ./pixi.toml

RUN pixi install -e ${CPU_OR_GPU} --frozen \
&& pixi clean cache --yes \
&& pixi info

COPY entrypoint.sh /entrypoint.sh
COPY --chown=${RUNTIME_USER}:${RUNTIME_USER} tests ./tests

CMD ["bash", "/entrypoint.sh"]
8 changes: 8 additions & 0 deletions runtime/Dockerfile-lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM ghcr.io/prefix-dev/pixi:0.26.1-jammy

USER root

RUN mkdir -p /tmp
WORKDIR /tmp

ENTRYPOINT ["pixi", "tree", "--manifest-path", "pixi.toml", "--platform", "linux-64", "-v"]
Loading
Loading