From 0b37ac23f1385807b2b4434727676fe7bbf7a8b8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 7 May 2024 14:55:04 -0400 Subject: [PATCH] Perspective neurodocker invocation + produced Dockerfile (broken) It is based on WIP done in https://github.com/ReproNim/neurodocker/pull/611 to add support for Gentoo. But it is just a dirty draft so produced docker file is not good. --- Dockerfile | 174 +++++++++++++++++++++ generate_container.sh | 19 ++- gentoo-portage/make.conf | 21 +++ gentoo-portage/package.accept_keywords/gen | 2 + gentoo-portage/package.mask/bugs | 2 + gentoo-portage/repos.conf/gentoo | 6 + gentoo-portage/repos.conf/science | 6 + 7 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 Dockerfile create mode 100644 gentoo-portage/make.conf create mode 100644 gentoo-portage/package.accept_keywords/gen create mode 100644 gentoo-portage/package.mask/bugs create mode 100644 gentoo-portage/repos.conf/gentoo create mode 100644 gentoo-portage/repos.conf/science diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0811d6f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,174 @@ +# Generated by Neurodocker and Reproenv. + +FROM docker.io/gentoo/portage:20240324 as portage +FROM docker.io/gentoo/stage3:20240318 +RUN COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo \ + && RUN emerge -v --noreplace dev-vcs/git \ + && RUN emerge -v1u portage \ + # Pinned commits for the dependency tree state + && ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 \ + && ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448 \ + && ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" \ + # This will be bound, and contents available outside of container + && RUN mkdir /outputs \ + && COPY gentoo-portage/ /etc/portage/ \ + # Moving gentoo repo from default rsync to git + && RUN rm /var/db/repos/gentoo -rf \ + # Cloning manually to prevent vdb update, pinning state via git + # Allegedly it's better to chain everything in one command, something with container layers 🤔 + && RUN \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \ + mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \ + git remote add origin ${REPO_URL} && \ + git fetch --filter="blob:none" origin $gentoo_hash && \ + git reset --hard $gentoo_hash && rm .git -rf && popd && \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \ + mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \ + git remote add origin ${REPO_URL} && \ + git fetch --filter="blob:none" origin $science_hash && \ + git reset --hard $science_hash && rm .git -rf && popd \ + # Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively + # Christian from the future: Maybe we want the option to re-sync if we're debugging it interactively... + #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "s/sync-type *= *git/sync-type =/g" + #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-uri/d" + #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-git-verify-commit-signature/d" + # Make sure all CPU flags supported by the hardware are whitelisted + # This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg. + # Removing it is safe, software will just not take full advantage of processor capabilities. + #RUN emerge cpuid2cpuflags + #RUN echo "*/* $(cpuid2cpuflags)" > /etc/portage/package.use/00cpu-flags + ### Emerge cool stuff here + ### Autounmask-continue enables all features on dependencies which the top level packages require + ### By default this needs user confirmation which would interrupt the build. +RUN emerge --autounmask-continue \ + afni \ + fsl \ + && rm -rf /var/tmp/portage/* +COPY ["environment.yml", \ + "/opt/environment.yml"] +COPY ["src", \ + "/opt/dsst-defacing-pipeline"] +ENV CONDA_DIR="/opt/miniconda-latest" \ + PATH="/opt/miniconda-latest/bin:$PATH" +RUN \ + # Install dependencies. + && export PATH="/opt/miniconda-latest/bin:$PATH" \ + && echo "Downloading Miniconda installer ..." \ + && conda_installer="/tmp/miniconda.sh" \ + && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && bash "$conda_installer" -b -p /opt/miniconda-latest \ + && rm -f "$conda_installer" \ + && conda update -yq -nbase conda \ + # Prefer packages in conda-forge + && conda config --system --prepend channels conda-forge \ + # Packages in lower-priority channels not considered if a package with the same + # name exists in a higher priority channel. Can dramatically speed up installations. + # Conda recommends this as a default + # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html + && conda config --set channel_priority strict \ + && conda config --system --set auto_update_conda false \ + && conda config --system --set show_channel_urls true \ + # Enable `conda activate` + && conda init bash \ + && conda env create --name dsstdeface --file /opt/environment.yml \ + # Clean up + && sync && conda clean --all --yes && sync \ + && rm -rf ~/.cache/pip/* +RUN test "$(getent passwd dsst)" \ + || useradd --no-user-group --create-home --shell /bin/bash dsst +USER dsst +ENTRYPOINT ["/opt/dsst-defacing-pipeline/run.py"] + +# Save specification to JSON. +USER root +RUN printf '{ \ + "pkg_manager": "portage", \ + "existing_users": [ \ + "root" \ + ], \ + "instructions": [ \ + { \ + "name": "from_", \ + "kwds": { \ + "base_image": "docker.io/gentoo/portage:20240324 as portage" \ + } \ + }, \ + { \ + "name": "from_", \ + "kwds": { \ + "base_image": "docker.io/gentoo/stage3:20240318" \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\\nRUN emerge -v --noreplace dev-vcs/git\\nRUN emerge -v1u portage\\n# Pinned commits for the dependency tree state\\nARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\\nARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\\nARG FEATURES=\\"-ipc-sandbox -network-sandbox -pid-sandbox\\"\\n# This will be bound, and contents available outside of container\\nRUN mkdir /outputs\\nCOPY gentoo-portage/ /etc/portage/\\n# Moving gentoo repo from default rsync to git\\nRUN rm /var/db/repos/gentoo -rf\\n# Cloning manually to prevent vdb update, pinning state via git\\n# Allegedly it'"'"'s better to chain everything in one command, something with container layers \\ud83e\\udd14\\nRUN \\\\\\n REPO_URL=$\(grep \\"^sync-uri\\" /etc/portage/repos.conf/gentoo | sed -e \\"s/sync-uri *= *//g\\"\) && \\\\\\n mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \\\\\\n git remote add origin ${REPO_URL} && \\\\\\n git fetch --filter=\\"blob:none\\" origin $gentoo_hash && \\\\\\n git reset --hard $gentoo_hash && rm .git -rf && popd && \\\\\\n REPO_URL=$\(grep \\"^sync-uri\\" /etc/portage/repos.conf/science | sed -e \\"s/sync-uri *= *//g\\"\) && \\\\\\n mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \\\\\\n git remote add origin ${REPO_URL} && \\\\\\n git fetch --filter=\\"blob:none\\" origin $science_hash && \\\\\\n git reset --hard $science_hash && rm .git -rf && popd\\n# Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively\\n# Christian from the future: Maybe we want the option to re-sync if we'"'"'re debugging it interactively...\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"s/sync-type *= *git/sync-type =/g\\"\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"/sync-uri/d\\"\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"/sync-git-verify-commit-signature/d\\"\\n# Make sure all CPU flags supported by the hardware are whitelisted\\n# This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg.\\n# Removing it is safe, software will just not take full advantage of processor capabilities.\\n#RUN emerge cpuid2cpuflags\\n#RUN echo \\"*/* $\(cpuid2cpuflags\)\\" > /etc/portage/package.use/00cpu-flags\\n### Emerge cool stuff here\\n### Autounmask-continue enables all features on dependencies which the top level packages require\\n### By default this needs user confirmation which would interrupt the build." \ + } \ + }, \ + { \ + "name": "install", \ + "kwds": { \ + "pkgs": [ \ + "afni", \ + "fsl" \ + ], \ + "opts": null \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "emerge --autounmask-continue \\\\\\n afni \\\\\\n fsl \\\\\\n && rm -rf /var/tmp/portage/*" \ + } \ + }, \ + { \ + "name": "copy", \ + "kwds": { \ + "source": [ \ + "environment.yml", \ + "/opt/environment.yml" \ + ], \ + "destination": "/opt/environment.yml" \ + } \ + }, \ + { \ + "name": "copy", \ + "kwds": { \ + "source": [ \ + "src", \ + "/opt/dsst-defacing-pipeline" \ + ], \ + "destination": "/opt/dsst-defacing-pipeline" \ + } \ + }, \ + { \ + "name": "env", \ + "kwds": { \ + "CONDA_DIR": "/opt/miniconda-latest", \ + "PATH": "/opt/miniconda-latest/bin:$PATH" \ + } \ + }, \ + { \ + "name": "run", \ + "kwds": { \ + "command": "\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda env create --name dsstdeface --file /opt/environment.yml\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ + } \ + }, \ + { \ + "name": "user", \ + "kwds": { \ + "user": "dsst" \ + } \ + }, \ + { \ + "name": "entrypoint", \ + "kwds": { \ + "args": [ \ + "/opt/dsst-defacing-pipeline/run.py" \ + ] \ + } \ + } \ + ] \ +}' > /.reproenv.json +USER dsst +# End saving to specification to JSON. diff --git a/generate_container.sh b/generate_container.sh index a0081ef..de045f9 100755 --- a/generate_container.sh +++ b/generate_container.sh @@ -6,26 +6,29 @@ generate() { # more details might come on https://github.com/ReproNim/neurodocker/issues/330 [ "$1" == singularity ] && add_entry=' "$@"' || add_entry='' #neurodocker generate "$1" \ - ndversion=0.9.5 + #ndversion=0.9.5 #ndversion=master #docker run --rm repronim/neurodocker:$ndversion \ # ATM needs devel version of neurodocker for a fix to AFNI recipe + #--base-image neurodebian:bookworm \ + #--ndfreeze date=20240320 \ + dest=/opt/dsst-defacing-pipeline neurodocker \ generate "$1" \ - --base-image neurodebian:bookworm \ - --ndfreeze date=20240320 \ + --pkg-manager portage \ + --base-image "docker.io/gentoo/portage:20240324 as portage" \ + --base-image "docker.io/gentoo/stage3:20240318" \ + --gentoo gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 \ + --install afni fsl \ --copy environment.yml /opt/environment.yml \ + --copy src "$dest" \ --miniconda \ version=latest \ env_name=dsstdeface \ env_exists=false \ yaml_file=/opt/environment.yml \ - --pkg-manager=apt \ - --install vim wget strace time ncdu gnupg curl procps pigz less tree \ - --run "apt-get update && apt-get -y dist-upgrade" \ - --afni method=binaries version=latest \ --user=dsst \ - --entrypoint "bash" + --entrypoint "$dest/run.py" #--run "curl -sL https://deb.nodesource.com/setup_16.x | bash - " \ #--install nodejs npm \ #--run "npm install -g bids-validator@1.14.4" \ diff --git a/gentoo-portage/make.conf b/gentoo-portage/make.conf new file mode 100644 index 0000000..5d2eded --- /dev/null +++ b/gentoo-portage/make.conf @@ -0,0 +1,21 @@ +### This file contains system-wide build variables, including Gentoo variables such as USE, which enable/disable optional package features. + +COMMON_FLAGS="-O2 -pipe -march=native" +# Comment the following out on systems with less than 8 threads +MAKEOPTS="--jobs 8 --load-average 9" +CFLAGS="${COMMON_FLAGS}" +CXXFLAGS="${COMMON_FLAGS}" +FCFLAGS="${COMMON_FLAGS}" +FFLAGS="${COMMON_FLAGS}" + +# NOTE: This stage was built with the bindist Use flag enabled + +# This sets the language of build output to English. +# Please keep this setting intact when reporting bugs. +LC_MESSAGES=C + +USE="${USE} science" +ACCEPT_LICENSE="*" + +# Needed in the container environment +#FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" diff --git a/gentoo-portage/package.accept_keywords/gen b/gentoo-portage/package.accept_keywords/gen new file mode 100644 index 0000000..467373b --- /dev/null +++ b/gentoo-portage/package.accept_keywords/gen @@ -0,0 +1,2 @@ +### This is needed because ::science packages are generally not marked as stable +*/* ~amd64 diff --git a/gentoo-portage/package.mask/bugs b/gentoo-portage/package.mask/bugs new file mode 100644 index 0000000..1e593eb --- /dev/null +++ b/gentoo-portage/package.mask/bugs @@ -0,0 +1,2 @@ +### This is empty, thankfully. +### If we find bugs in some version of some package we can blacklist the package, version, or feature that causes it here. diff --git a/gentoo-portage/repos.conf/gentoo b/gentoo-portage/repos.conf/gentoo new file mode 100644 index 0000000..5d53ee4 --- /dev/null +++ b/gentoo-portage/repos.conf/gentoo @@ -0,0 +1,6 @@ +[gentoo] +location = /var/db/repos/gentoo +# We sync manually, but we need sync-uri to be written down somewhere to do so +sync-type = git +sync-uri = https://anongit.gentoo.org/git/repo/gentoo.git +sync-git-verify-commit-signature = yes diff --git a/gentoo-portage/repos.conf/science b/gentoo-portage/repos.conf/science new file mode 100644 index 0000000..4a03df2 --- /dev/null +++ b/gentoo-portage/repos.conf/science @@ -0,0 +1,6 @@ +[science] +location = /var/db/repos/science +# We sync manually, but we need sync-uri to be written down somewhere to do so +sync-type = git +sync-uri = https://anongit.gentoo.org/git/proj/sci.git +priority = 7777