Use Pixi for dependency management

.github/workflows/build.yml

@@ -41,6 +41,19 @@ jobs:
  LATEST_TAG: ${{ matrix.proc }}-latest
 
  steps:
+ - name: Remove unwanted software
+ run: |
+ echo "Available storage before:"
+ sudo df -h
+ echo
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /usr/local/lib/android
+ sudo rm -rf /opt/ghc
+ sudo rm -rf /opt/hostedtoolcache/CodeQL
+ echo "Available storage after:"
+ sudo df -h
+ echo
+
  - uses: actions/checkout@v4
 
  - name: Build Image
@@ -56,7 +69,9 @@ jobs:
 
  - name: Tests packages in container
  run: |
- docker run $LOGIN_SERVER/$IMAGE:$SHA_TAG python -m pytest tests
+ docker run --network none \
+ $LOGIN_SERVER/$IMAGE:$SHA_TAG \
+ /code_execution/.pixi/envs/${{ matrix.proc }}/bin/python -m pytest tests
 
  - name: Log into Azure
  if: ${{ fromJson(env.SHOULD_PUBLISH) }}
@@ -78,4 +93,3 @@ jobs:
  if: ${{ fromJson(env.SHOULD_PUBLISH) }}
  run: |
  docker push $LOGIN_SERVER/$IMAGE --all-tags
-

Makefile

@@ -8,6 +8,8 @@ else
 CPU_OR_GPU ?= gpu
 endif
 
+BLOCK_INTERNET ?= true
+
 TAG := ${CPU_OR_GPU}-latest
 LOCAL_TAG := ${CPU_OR_GPU}-local
 
@@ -109,61 +111,42 @@ endif
 #################################################################################
 # Commands for building the container if you are changing the requirements #
 #################################################################################
-.PHONY: build clean interact-container pack-example pack-submission pull test-container test-submission update-lockfiles
+.PHONY: build clean interact-container pack-example pack-submission pull test-container test-submission update-lockfile
 
 ## Builds the container locally
 build:
  docker build runtime \
  --build-arg CPU_OR_GPU=${CPU_OR_GPU} \
  --tag ${LOCAL_IMAGE}:${LOCAL_TAG}
 
-
-runtime/conda-lock-cpu.yml: runtime/environment-cpu.yml
- @echo Locking the CPU environment
- conda-lock \
- --mamba \
- -p linux-64 \
- --without-cuda \
- -f runtime/environment-cpu.yml \
- --lockfile runtime/conda-lock-cpu.yml
- conda-lock render \
- -p linux-64 \
- --filename-template runtime/environment-cpu.lock \
- runtime/conda-lock-cpu.yml
-
-runtime/conda-lock-gpu.yml: runtime/environment-gpu.yml
- @echo Locking the GPU environment
- conda-lock \
- --mamba \
- -p linux-64 \
- --with-cuda 11.8 \
- -f runtime/environment-gpu.yml \
- --lockfile runtime/conda-lock-gpu.yml
- conda-lock render \
- -p linux-64 \
- --filename-template runtime/environment-gpu.lock \
- runtime/conda-lock-gpu.yml
-
-## Updates runtime environment lockfiles
-update-lockfiles: runtime/conda-lock-cpu.yml runtime/conda-lock-gpu.yml
- @python runtime/tests/test_lockfile.py runtime/conda-lock-cpu.yml
- @python runtime/tests/test_lockfile.py runtime/conda-lock-gpu.yml
-
+## Updates runtime environment lockfile using Docker
+update-lockfile:
+ @echo Generating the lockfile for CPU and GPU within Docker
+ docker build runtime \
+ --file runtime/Dockerfile-lock \
+ --tag pixi-lock:local
+ @echo Running lock container
+ docker run \
+ --mount type=bind,source="$(shell pwd)"/runtime,target=/tmp \
+ --rm \
+ pixi-lock:local
 
 ## Ensures that your locally built image can import all the Python packages successfully when it runs
 test-container: _check_image _echo_image _submission_write_perms
  docker run \
  ${GPU_ARGS} \
+ ${NETWORK_ARGS} \
  ${TTY_ARGS} \
- --mount type=bind,source="$(shell pwd)"/runtime/tests,target=/tests,readonly \
  --pid host \
  ${SUBMISSION_IMAGE_ID} \
- python -m pytest -v tests
+ pixi run -e ${CPU_OR_GPU} python -m pytest tests
+
 
 ## Open an interactive bash shell within the running container (with network access)
 interact-container: _check_image _echo_image _submission_write_perms
  docker run \
  ${GPU_ARGS} \
+ ${NETWORK_ARGS} \
  --mount type=bind,source=${shell pwd}/data,target=/code_execution/data,readonly \
  --mount type=bind,source="$(shell pwd)/submission",target=/code_execution/submission \
  --shm-size 8g \

runtime/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 mambaorg/micromamba:1.5.3-bookworm-slim
+FROM ghcr.io/prefix-dev/pixi:0.26.1-jammy-cuda-11.8.0
 
 USER root
 
@@ -10,24 +10,39 @@ ENV DEBIAN_FRONTEND=noninteractive \
  PYTHONUNBUFFERED=1 \
  SHELL=/bin/bash
 
+# Create user andset permissions
+ENV RUNTIME_USER=runtimeuser
+ENV RUNTIME_UID=1000
+ENV RUNTIME_GID=1000
+ENV CPU_OR_GPU=$CPU_OR_GPU
+
+RUN echo "Creating ${RUNTIME_USER} user..." \
+ && groupadd --gid ${RUNTIME_GID} ${RUNTIME_USER} \
+ && useradd --create-home --gid ${RUNTIME_GID} --no-log-init --uid ${RUNTIME_UID} ${RUNTIME_USER}
+
 COPY apt.txt apt.txt
 RUN apt-get update --fix-missing \
  && apt-get install -y apt-utils 2> /dev/null \
  && xargs -a apt.txt apt-get install -y \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* /apt.txt
 
-COPY --chown=$MAMBA_USER:$MAMBA_USER conda-lock-${CPU_OR_GPU}.yml /tmp/conda-lock.yml
-RUN micromamba install --name base --yes --file /tmp/conda-lock.yml && \
- micromamba clean --all --force-pkgs-dirs --yes
-
+# Set up code execution working directory
 RUN mkdir /code_execution
-RUN chown -R ${MAMBA_USER}:${MAMBA_USER} /code_execution
+RUN chown -R ${RUNTIME_USER}:${RUNTIME_USER} /code_execution
+WORKDIR /code_execution
 
-COPY tests /code_execution/tests
-COPY entrypoint.sh /entrypoint.sh
+# Switch to runtime user
+USER ${RUNTIME_USER}
 
-WORKDIR /code_execution
-USER ${MAMBA_USER}
+COPY pixi.lock ./pixi.lock
+COPY pixi.toml ./pixi.toml
+
+RUN pixi install -e ${CPU_OR_GPU} --frozen \
+ && pixi clean cache --yes \
+ && pixi info
+
+COPY entrypoint.sh /entrypoint.sh
+COPY --chown=${RUNTIME_USER}:${RUNTIME_USER} tests ./tests
 
 CMD ["bash", "/entrypoint.sh"]

runtime/Dockerfile-lock

@@ -0,0 +1,8 @@
+FROM ghcr.io/prefix-dev/pixi:0.26.1-bookworm-slim
+
+USER root
+
+RUN mkdir -p /tmp
+WORKDIR /tmp
+
+ENTRYPOINT ["pixi", "tree", "--manifest-path", "pixi.toml", "--platform", "linux-64", "-v"]