Perspective neurodocker invocation + produced Dockerfile (broken)

It is based on WIP done in ReproNim/neurodocker#611 to add support for Gentoo. But it is just a dirty draft so produced docker file is not good.
yarikoptic · May 7, 2024 · 0b37ac2 · 0b37ac2
1 parent 85b0be6
commit 0b37ac2
Show file tree

Hide file tree

Showing 7 changed files with 222 additions and 8 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,174 @@
+# Generated by Neurodocker and Reproenv.
+
+FROM docker.io/gentoo/portage:20240324 as portage
+FROM docker.io/gentoo/stage3:20240318
+RUN COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo \
+    && RUN emerge -v --noreplace dev-vcs/git \
+    && RUN emerge -v1u portage \
+    # Pinned commits for the dependency tree state
+    && ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 \
+    && ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448 \
+    && ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox" \
+    # This will be bound, and contents available outside of container
+    && RUN mkdir /outputs \
+    && COPY gentoo-portage/ /etc/portage/ \
+    # Moving gentoo repo from default rsync to git
+    && RUN rm /var/db/repos/gentoo -rf \
+    # Cloning manually to prevent vdb update, pinning state via git
+    # Allegedly it's better to chain everything in one command, something with container layers 🤔
+    && RUN \
+               REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \
+               mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \
+                       git remote add origin ${REPO_URL} && \
+                       git fetch --filter="blob:none" origin $gentoo_hash && \
+                       git reset --hard $gentoo_hash && rm .git -rf && popd && \
+               REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \
+               mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \
+                       git remote add origin ${REPO_URL} && \
+                       git fetch --filter="blob:none" origin $science_hash && \
+                       git reset --hard $science_hash && rm .git -rf && popd \
+    # Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively
+    # Christian from the future: Maybe we want the option to re-sync if we're debugging it interactively...
+    #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "s/sync-type *= *git/sync-type =/g"
+    #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-uri/d"
+    #RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e "/sync-git-verify-commit-signature/d"
+    # Make sure all CPU flags supported by the hardware are whitelisted
+    # This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg.
+    # Removing it is safe, software will just not take full advantage of processor capabilities.
+    #RUN emerge cpuid2cpuflags
+    #RUN echo "*/* $(cpuid2cpuflags)" > /etc/portage/package.use/00cpu-flags
+    ### Emerge cool stuff here
+    ### Autounmask-continue enables all features on dependencies which the top level packages require
+    ### By default this needs user confirmation which would interrupt the build.
+RUN emerge --autounmask-continue \
+                  afni \
+                  fsl \
+           && rm -rf /var/tmp/portage/*
+COPY ["environment.yml", \
+      "/opt/environment.yml"]
+COPY ["src", \
+      "/opt/dsst-defacing-pipeline"]
+ENV CONDA_DIR="/opt/miniconda-latest" \
+    PATH="/opt/miniconda-latest/bin:$PATH"
+RUN \
+    # Install dependencies.
+    && export PATH="/opt/miniconda-latest/bin:$PATH" \
+    && echo "Downloading Miniconda installer ..." \
+    && conda_installer="/tmp/miniconda.sh" \
+    && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && bash "$conda_installer" -b -p /opt/miniconda-latest \
+    && rm -f "$conda_installer" \
+    && conda update -yq -nbase conda \
+    # Prefer packages in conda-forge
+    && conda config --system --prepend channels conda-forge \
+    # Packages in lower-priority channels not considered if a package with the same
+    # name exists in a higher priority channel. Can dramatically speed up installations.
+    # Conda recommends this as a default
+    # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html
+    && conda config --set channel_priority strict \
+    && conda config --system --set auto_update_conda false \
+    && conda config --system --set show_channel_urls true \
+    # Enable `conda activate`
+    && conda init bash \
+    && conda env create  --name dsstdeface --file /opt/environment.yml \
+    # Clean up
+    && sync && conda clean --all --yes && sync \
+    && rm -rf ~/.cache/pip/*
+RUN test "$(getent passwd dsst)" \
+    || useradd --no-user-group --create-home --shell /bin/bash dsst
+USER dsst
+ENTRYPOINT ["/opt/dsst-defacing-pipeline/run.py"]
+
+# Save specification to JSON.
+USER root
+RUN printf '{ \
+  "pkg_manager": "portage", \
+  "existing_users": [ \
+    "root" \
+  ], \
+  "instructions": [ \
+    { \
+      "name": "from_", \
+      "kwds": { \
+        "base_image": "docker.io/gentoo/portage:20240324 as portage" \
+      } \
+    }, \
+    { \
+      "name": "from_", \
+      "kwds": { \
+        "base_image": "docker.io/gentoo/stage3:20240318" \
+      } \
+    }, \
+    { \
+      "name": "run", \
+      "kwds": { \
+        "command": "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\\nRUN emerge -v --noreplace dev-vcs/git\\nRUN emerge -v1u portage\\n# Pinned commits for the dependency tree state\\nARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\\nARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\\nARG FEATURES=\\"-ipc-sandbox -network-sandbox -pid-sandbox\\"\\n# This will be bound, and contents available outside of container\\nRUN mkdir /outputs\\nCOPY gentoo-portage/ /etc/portage/\\n# Moving gentoo repo from default rsync to git\\nRUN rm /var/db/repos/gentoo -rf\\n# Cloning manually to prevent vdb update, pinning state via git\\n# Allegedly it'"'"'s better to chain everything in one command, something with container layers \\ud83e\\udd14\\nRUN \\\\\\n        REPO_URL=$\(grep \\"^sync-uri\\" /etc/portage/repos.conf/gentoo | sed -e \\"s/sync-uri *= *//g\\"\) && \\\\\\n        mkdir -p /var/db/repos/gentoo && pushd /var/db/repos/gentoo && git init . && \\\\\\n                git remote add origin ${REPO_URL} && \\\\\\n                git fetch --filter=\\"blob:none\\" origin $gentoo_hash && \\\\\\n                git reset --hard $gentoo_hash && rm .git -rf && popd && \\\\\\n        REPO_URL=$\(grep \\"^sync-uri\\" /etc/portage/repos.conf/science | sed -e \\"s/sync-uri *= *//g\\"\) && \\\\\\n        mkdir -p /var/db/repos/science && pushd /var/db/repos/science && git init . && \\\\\\n                git remote add origin ${REPO_URL} && \\\\\\n                git fetch --filter=\\"blob:none\\" origin $science_hash && \\\\\\n                git reset --hard $science_hash && rm .git -rf && popd\\n# Old Christian: Remove sync-uri to not accidentally re-sync if we work with the package management interactively\\n# Christian from the future: Maybe we want the option to re-sync if we'"'"'re debugging it interactively...\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"s/sync-type *= *git/sync-type =/g\\"\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"/sync-uri/d\\"\\n#RUN sed -i /etc/portage/repos.conf/{gentoo,science} -e \\"/sync-git-verify-commit-signature/d\\"\\n# Make sure all CPU flags supported by the hardware are whitelisted\\n# This only affects packages with handwritten assembly language optimizations, e.g. ffmpeg.\\n# Removing it is safe, software will just not take full advantage of processor capabilities.\\n#RUN emerge cpuid2cpuflags\\n#RUN echo \\"*/* $\(cpuid2cpuflags\)\\" > /etc/portage/package.use/00cpu-flags\\n### Emerge cool stuff here\\n### Autounmask-continue enables all features on dependencies which the top level packages require\\n### By default this needs user confirmation which would interrupt the build." \
+      } \
+    }, \
+    { \
+      "name": "install", \
+      "kwds": { \
+        "pkgs": [ \
+          "afni", \
+          "fsl" \
+        ], \
+        "opts": null \
+      } \
+    }, \
+    { \
+      "name": "run", \
+      "kwds": { \
+        "command": "emerge --autounmask-continue \\\\\\n           afni \\\\\\n           fsl \\\\\\n    && rm -rf /var/tmp/portage/*" \
+      } \
+    }, \
+    { \
+      "name": "copy", \
+      "kwds": { \
+        "source": [ \
+          "environment.yml", \
+          "/opt/environment.yml" \
+        ], \
+        "destination": "/opt/environment.yml" \
+      } \
+    }, \
+    { \
+      "name": "copy", \
+      "kwds": { \
+        "source": [ \
+          "src", \
+          "/opt/dsst-defacing-pipeline" \
+        ], \
+        "destination": "/opt/dsst-defacing-pipeline" \
+      } \
+    }, \
+    { \
+      "name": "env", \
+      "kwds": { \
+        "CONDA_DIR": "/opt/miniconda-latest", \
+        "PATH": "/opt/miniconda-latest/bin:$PATH" \
+      } \
+    }, \
+    { \
+      "name": "run", \
+      "kwds": { \
+        "command": "\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-latest/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-latest\\nrm -f \\"$conda_installer\\"\\nconda update -yq -nbase conda\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda env create  --name dsstdeface --file /opt/environment.yml\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \
+      } \
+    }, \
+    { \
+      "name": "user", \
+      "kwds": { \
+        "user": "dsst" \
+      } \
+    }, \
+    { \
+      "name": "entrypoint", \
+      "kwds": { \
+        "args": [ \
+          "/opt/dsst-defacing-pipeline/run.py" \
+        ] \
+      } \
+    } \
+  ] \
+}' > /.reproenv.json
+USER dsst
+# End saving to specification to JSON.
diff --git a/generate_container.sh b/generate_container.sh
@@ -6,26 +6,29 @@ generate() {
 	# more details might come on https://github.com/ReproNim/neurodocker/issues/330
 	[ "$1" == singularity ] && add_entry=' "$@"' || add_entry=''
 	#neurodocker generate "$1" \
-	ndversion=0.9.5
+	#ndversion=0.9.5
 	#ndversion=master
 	#docker run --rm repronim/neurodocker:$ndversion \
 	# ATM needs devel version  of neurodocker for a fix to AFNI recipe
+	#--base-image neurodebian:bookworm \
+	#--ndfreeze date=20240320 \
+	dest=/opt/dsst-defacing-pipeline
 	neurodocker \
 		generate "$1" \
-		--base-image neurodebian:bookworm \
-		--ndfreeze date=20240320 \
+		--pkg-manager portage \
+		--base-image "docker.io/gentoo/portage:20240324 as portage" \
+		--base-image "docker.io/gentoo/stage3:20240318" \
+		--gentoo gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6 \
+		--install afni fsl \
 		--copy environment.yml /opt/environment.yml \
+		--copy src "$dest" \
 		--miniconda \
 			version=latest \
 			env_name=dsstdeface \
 			env_exists=false \
 			yaml_file=/opt/environment.yml \
-		--pkg-manager=apt \
-		--install vim wget strace time ncdu gnupg curl procps pigz less tree \
-		--run "apt-get update && apt-get -y dist-upgrade" \
-		--afni method=binaries version=latest \
 		--user=dsst \
-		--entrypoint "bash"
+		--entrypoint "$dest/run.py"
 		#--run "curl -sL https://deb.nodesource.com/setup_16.x | bash - " \
 		#--install nodejs npm \
 		#--run "npm install -g [email protected]" \

diff --git a/gentoo-portage/make.conf b/gentoo-portage/make.conf
@@ -0,0 +1,21 @@
+### This file contains system-wide build variables, including Gentoo variables such as USE, which enable/disable optional package features.
+
+COMMON_FLAGS="-O2 -pipe -march=native"
+# Comment the following out on systems with less than 8 threads
+MAKEOPTS="--jobs 8 --load-average 9"
+CFLAGS="${COMMON_FLAGS}"
+CXXFLAGS="${COMMON_FLAGS}"
+FCFLAGS="${COMMON_FLAGS}"
+FFLAGS="${COMMON_FLAGS}"
+
+# NOTE: This stage was built with the bindist Use flag enabled
+
+# This sets the language of build output to English.
+# Please keep this setting intact when reporting bugs.
+LC_MESSAGES=C
+
+USE="${USE} science"
+ACCEPT_LICENSE="*"
+
+# Needed in the container environment
+#FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox"
diff --git a/gentoo-portage/package.accept_keywords/gen b/gentoo-portage/package.accept_keywords/gen
@@ -0,0 +1,2 @@
+### This is needed because ::science packages are generally not marked as stable
+*/* ~amd64
diff --git a/gentoo-portage/package.mask/bugs b/gentoo-portage/package.mask/bugs
@@ -0,0 +1,2 @@
+### This is empty, thankfully.
+### If we find bugs in some version of some package we can blacklist the package, version, or feature that causes it here.
diff --git a/gentoo-portage/repos.conf/gentoo b/gentoo-portage/repos.conf/gentoo
@@ -0,0 +1,6 @@
+[gentoo]
+location = /var/db/repos/gentoo
+# We sync manually, but we need sync-uri to be written down somewhere to do so
+sync-type = git
+sync-uri = https://anongit.gentoo.org/git/repo/gentoo.git
+sync-git-verify-commit-signature = yes
diff --git a/gentoo-portage/repos.conf/science b/gentoo-portage/repos.conf/science
@@ -0,0 +1,6 @@
+[science]
+location = /var/db/repos/science
+# We sync manually, but we need sync-uri to be written down somewhere to do so
+sync-type = git
+sync-uri = https://anongit.gentoo.org/git/proj/sci.git
+priority = 7777