From 3604d6554155a6957ae881ed590ee303d1247489 Mon Sep 17 00:00:00 2001 From: Leonhard Hennig Date: Tue, 2 Jul 2024 08:31:25 +0200 Subject: [PATCH 1/5] Update folder locations (resolves https://github.com/DFKI-NLP/pegasus-bridle/issues/11) --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index b830d5f..cfb50ab 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ # choose an Enroot Image # IMPORTANT: For the scripts provided here to function correctly, there needs to be conda installed in that image! -CONTAINER_IMAGE=/netscratch/enroot/nvcr.io_nvidia_pytorch_23.07-py3.sqsh +CONTAINER_IMAGE=/enroot/nvcr.io_nvidia_pytorch_23.07-py3.sqsh # chooses a Slurm partition PARTITION=batch @@ -38,7 +38,7 @@ HOST_CONDA_ENVS_DIR=/netscratch/$USER/miniconda3 # change this if you use another Enroot image with a different conda location CONTAINER_CONDA_ENVS_DIR=/opt/conda # assemble everything: -CONTAINER_MOUNTS=/netscratch:/netscratch,/ds:/ds:ro,"$HOST_WORKDIR":"$CONTAINER_WORKDIR","$HOST_CONDA_ENVS_DIR":"$CONTAINER_CONDA_ENVS_DIR","$HOST_CACHEDIR":"/home/$USER/.cache","$HOST_CACHEDIR":/root/.cache,"$HOST_CACHEDIR":/home/root/.cache +CONTAINER_MOUNTS=/netscratch:/netscratch,/fscratch/$USER:/fscratch/$USER,/ds:/ds:ro,"$HOST_WORKDIR":"$CONTAINER_WORKDIR","$HOST_CONDA_ENVS_DIR":"$CONTAINER_CONDA_ENVS_DIR","$HOST_CACHEDIR":"/home/$USER/.cache","$HOST_CACHEDIR":/root/.cache,"$HOST_CACHEDIR":/home/root/.cache # Define which conda environment should be activated. Set to $CONDA_DEFAULT_ENV, to use the one that is activated at # the host. If not defined, no conda environment will be activated and the base python interpreter will be used. From 074c9a35081afd3bc0dc958f0e14c133d73d0e0b Mon Sep 17 00:00:00 2001 From: Leonhard Hennig Date: Tue, 2 Jul 2024 10:01:16 +0200 Subject: [PATCH 2/5] add conda install --- activate_and_execute.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/activate_and_execute.sh b/activate_and_execute.sh index a8bf547..4b8931b 100644 --- a/activate_and_execute.sh +++ b/activate_and_execute.sh @@ -6,6 +6,28 @@ SCRIPT_DIR=$(dirname $0) ARGS=( "$@" ) shift $# +if ! command -v conda > /dev/null 2>&1; then + echo "Installing conda..." + #wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O ~/miniconda.sh + #bash ~/miniconda.sh -b -p $HOME/miniconda + # Following https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html + # Install our public GPG key to trusted store + curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg + install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg + # Check whether fingerprint is correct (will output an error message otherwise) + gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 + # Add our Debian repo + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list + apt update + apt install conda + source /opt/conda/etc/profile.d/conda.sh + conda -V + echo "channels:" > /opt/conda/.condarc + echo " - conda-forge" >> /opt/conda/.condarc + echo "channel_priority: strict" >> /opt/conda/.condarc +fi + + # activate conda environment, if the variable is defined if [ -n "$CONDA_ENV" ]; then echo "activate conda environment: $CONDA_ENV" From d2a45d6e7bca6480386301319343ec5154908820 Mon Sep 17 00:00:00 2001 From: Leonhard Hennig Date: Tue, 2 Jul 2024 10:12:02 +0200 Subject: [PATCH 3/5] updated conda install --- activate_and_execute.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/activate_and_execute.sh b/activate_and_execute.sh index 4b8931b..865daf1 100644 --- a/activate_and_execute.sh +++ b/activate_and_execute.sh @@ -18,10 +18,9 @@ if ! command -v conda > /dev/null 2>&1; then gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 # Add our Debian repo echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list - apt update - apt install conda + apt -qy update + apt -qy install conda source /opt/conda/etc/profile.d/conda.sh - conda -V echo "channels:" > /opt/conda/.condarc echo " - conda-forge" >> /opt/conda/.condarc echo "channel_priority: strict" >> /opt/conda/.condarc From a8b63857393c61e48c55085806d975927de3f76f Mon Sep 17 00:00:00 2001 From: Leonhard Hennig Date: Tue, 2 Jul 2024 10:14:36 +0200 Subject: [PATCH 4/5] updated conda install --- activate_and_execute.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/activate_and_execute.sh b/activate_and_execute.sh index 865daf1..1f82b2e 100644 --- a/activate_and_execute.sh +++ b/activate_and_execute.sh @@ -7,10 +7,8 @@ ARGS=( "$@" ) shift $# if ! command -v conda > /dev/null 2>&1; then - echo "Installing conda..." - #wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O ~/miniconda.sh - #bash ~/miniconda.sh -b -p $HOME/miniconda # Following https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html + echo "Installing conda..." # Install our public GPG key to trusted store curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg From 4a0ecd76d7507e1f7d57b296c38f56920daf5477 Mon Sep 17 00:00:00 2001 From: Leonhard Hennig Date: Tue, 2 Jul 2024 10:17:23 +0200 Subject: [PATCH 5/5] updated conda install --- activate_and_execute.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activate_and_execute.sh b/activate_and_execute.sh index 1f82b2e..2de1562 100644 --- a/activate_and_execute.sh +++ b/activate_and_execute.sh @@ -16,8 +16,8 @@ if ! command -v conda > /dev/null 2>&1; then gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 # Add our Debian repo echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list - apt -qy update - apt -qy install conda + apt update + apt install conda source /opt/conda/etc/profile.d/conda.sh echo "channels:" > /opt/conda/.condarc echo " - conda-forge" >> /opt/conda/.condarc