From e3f329225fb0d81e11ed9de244a276bbbf6bc3a0 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 8 Jul 2024 16:22:37 +0200 Subject: [PATCH 01/52] start venv docs --- config/templates/hpc.template | 1 + .../setting_up_python_virtual_environments.md | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 mkdocs/docs/HPC/setting_up_python_virtual_environments.md diff --git a/config/templates/hpc.template b/config/templates/hpc.template index 5e3a1807199..2d98c38ffdb 100644 --- a/config/templates/hpc.template +++ b/config/templates/hpc.template @@ -23,6 +23,7 @@ nav: - Troubleshooting: troubleshooting.md - HPC Policies: sites/hpc_policies.md - Advanced topics: + - Setting Up Python Virtual Environments: setting_up_python_virtual_environments.md - Fine-tuning Job Specifications: fine_tuning_job_specifications.md - Multi-job submission: multi_job_submission.md - Compiling and testing your software on the HPC: compiling_your_software.md diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md new file mode 100644 index 00000000000..88143fe019d --- /dev/null +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -0,0 +1,66 @@ +# Python Virtual Environments (venv's) + +## Introduction + +Python virtual environments (venv's) are a way to create isolated Python environments. +You can install packages in a venv without affecting the system Python installation. +This is useful when you need to use a package that is not available as a module on the HPC cluster. + +## Creating a venv + +A venv can be created with the following commands: + +```bash +$ python -m venv myenv # Create a new venv named myenv +$ source myenv/bin/activate # Activate the venv +``` + +After activating the venv, you can install packages with `pip`: + +```bash +$ pip install example_package1 +$ pip install example_package2 +``` + +It is now possible to run Python scripts that use the installed packages in the venv. To deactivate the venv, run: + +```bash +$ deactivate +``` + +!!! note + Always prefer to use modules if they are available. + Modules are compiled and optimized for the HPC cluster, while packages installed with `pip` are not. + To check if a package is available as a module, you can use the following command: + + ```bash + $ module av package_name + ``` + +## Combining venv's with modules + +You can combine packages installed in a venv with modules. In the following example, +we load a PyTorch package as a module and install Poutyne in a venv: + +```bash +$ ml PyTorch/2.1.2-foss-2023a +$ python -m venv myenv +$ source myenv/bin/activate +$ pip install Poutyne +``` + +You can now use PyTorch and Poutyne in the same Python script: + +```python +import torch +import poutyne + +... +``` + +!!! warning + Activating a venv that was creating on a different cluster can cause issues. + This is because the binaries placed in the venv on creation at cluster `A` might not be compatible with the CPU architecture of cluster `B`. + To be safe, always create the virtual environment for the cluster you will be activating it on. + +## Creating a virtual environment on a specific cluster From b70a22ca9d802352c17bb5cda7f8f1695f64b34d Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 9 Jul 2024 10:58:06 +0200 Subject: [PATCH 02/52] Using virtual environments in job scripts --- .../setting_up_python_virtual_environments.md | 62 ++++++++++++++++--- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 88143fe019d..f6b87dbba64 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -39,8 +39,17 @@ $ deactivate ## Combining venv's with modules -You can combine packages installed in a venv with modules. In the following example, -we load a PyTorch package as a module and install Poutyne in a venv: +You can combine packages installed in a venv with modules. The following script uses +pytorch (which is available as a module) and Poutyne (which is not available as a module): + +```python title="pytorch_poutine_example.py" +import torch +import poutyne + +... +``` + +We load a PyTorch package as a module and install Poutyne in a venv: ```bash $ ml PyTorch/2.1.2-foss-2023a @@ -49,18 +58,53 @@ $ source myenv/bin/activate $ pip install Poutyne ``` -You can now use PyTorch and Poutyne in the same Python script: +While in the virtual environment, we can run the script without any issues: -```python -import torch -import poutyne - -... +```bash +$ python pytorch_poutine_example.py ``` !!! warning Activating a venv that was creating on a different cluster can cause issues. This is because the binaries placed in the venv on creation at cluster `A` might not be compatible with the CPU architecture of cluster `B`. - To be safe, always create the virtual environment for the cluster you will be activating it on. + // TODO: Give an example of this + ## Creating a virtual environment on a specific cluster + +To create a virtual environment for a specific cluster, you need to start an interactive shell on that cluster. +Let's say you want to create a virtual environment on the `skitty` cluster. + +```bash +$ module swap cluster/skitty +$ qsub -I +``` + +After some time, a shell will be started on the `skitty` cluster. +You can now create a virtual environment as described in [this section](#creating-a-venv). +This virtual environment can be used by jobs running on the `skitty` cluster. + +## Using virtual environments in job scripts + +After creating a virtual environment for a cluster, you need to activate it in your job script. + +```bash title="jobscript.pbs" +#!/bin/bash + +# Basic parameters +#PBS -N jobname ## Job name +#PBS -l nodes=1:ppn=2 ## 1 node, 2 processors per node (ppn=all to get a full node) +#PBS -l walltime=01:00:00 ## Max time your job will run (no more than 72:00:00) + +module load [module] +module load [module] + +cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted + +source myenv/bin/activate # Activate the virtual environment +python myscript.py # Run your Python script, or any other command within the virtual environment +deactivate # Deactivate the virtual environment +``` + +// TODO troubleshooting for: python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory. +// Happens when making an env with a python module loaded an entering the env without that module bein loaded anymore. From cb4c7c43d0a9e65bb2b74307835cd978a9b8c176 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Wed, 10 Jul 2024 14:37:03 +0200 Subject: [PATCH 03/52] added troubleshooting section --- .../setting_up_python_virtual_environments.md | 78 +++++++++++++++++-- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index f6b87dbba64..99316f0decd 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -65,9 +65,26 @@ $ python pytorch_poutine_example.py ``` !!! warning - Activating a venv that was creating on a different cluster can cause issues. - This is because the binaries placed in the venv on creation at cluster `A` might not be compatible with the CPU architecture of cluster `B`. - // TODO: Give an example of this + Activating a virtual environment (venv) created on a different cluster can cause issues. + This happens because the binaries in the venv from cluster A might not work with the CPU architecture of cluster B. + + For example, if we create a venv on the skitty cluster, + + ```bash + $ module swap cluster/skitty + $ qsub -I + $ python -m venv myenv + ``` + + return to the login node by pressing CTRL+D and try to use the virtual environment: + + ```bash + $ source myenv/bin/activate + $ python + Illegal instruction (core dumped) + ``` + + we are presented with the illigal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) ## Creating a virtual environment on a specific cluster @@ -106,5 +123,56 @@ python myscript.py # Run your Python script, or any other command within deactivate # Deactivate the virtual environment ``` -// TODO troubleshooting for: python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory. -// Happens when making an env with a python module loaded an entering the env without that module bein loaded anymore. +## Troubleshooting + +### Error: cannot open shared object file: No such file or directory + +There are two main reasons why this error could occur. + +1. you have not loaded the python module that was used to create the virtual environment. +2. you added or removed modules while in the virtual environment. + +#### Entering a virtual environment while the python module used to create it is not active + +When you load a python module and use that to make a virtual environment, you need to make sure that the same module +is loaded when you enter the environment. This is because the virtual environment keeps a reference to the base python +used to create it. + +The following commands illustrate this issue: + +```bash +$ module load Python/3.10.8-GCCcore-12.2.0 # Load a python module +$ python -m venv myenv # Create a virtual environment with loaded python module +$ module purge # Remove all loaded modules +$ source myenv/bin/activate # Activate the virtual environment +$ python # Start python +python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory +``` + +Here, the virtual environment tries to use the python module that was loaded when the environment was created, which is no longer available. +The solution is to load the same python module before activating the virtual environment: + +```bash +$ module load Python/3.10.8-GCCcore-12.2.0 # Load the same python module +$ source myenv/bin/activate # Activate the virtual environment +``` + +#### modifying modules while in a virtual environment + +You must not delete or add modules while in a virtual environment. +Adding and removing modules modifies the `$PATH` variable in the current shell. When activating a virtual environment, +it will store the `$PATH` variable of the shell at that moment. If you modify the `$PATH` variable while in a virtual environment by loading or deleting modules, +and deactivate the virtual environment, the `$PATH` variable will be reset to the one stored in the virtual environment. +trying to use those modules will lead to errors: + +```bash +$ module load Python/3.10.8-GCCcore-12.2.0 # Load a python module +$ python -m venv myenv # Create a virtual environment +$ source myenv/bin/activate # Activate the virtual environment (saves state of $PATH) +$ module purge # Unload all modules (modifies the $PATH) +$ deactivate # Deactivate the virtual environment (resets $PATH to saved state) +$ python # PATH contains a reference to the unloaded module +python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory +``` + +The solution is to only modify modules when not in a virtual environment. \ No newline at end of file From e840955fc136850fc7a477a08550903bb0adb5fc Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 16 Jul 2024 15:53:30 +0200 Subject: [PATCH 04/52] fixed illigal typo --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 99316f0decd..bf58cad418f 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -84,7 +84,7 @@ $ python pytorch_poutine_example.py Illegal instruction (core dumped) ``` - we are presented with the illigal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) + we are presented with the illegal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) ## Creating a virtual environment on a specific cluster From e6b57d4ae6b615cfdf9b953b5147ff69113c77cf Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 16 Jul 2024 16:03:13 +0200 Subject: [PATCH 05/52] made link to creating venv section clearer --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index bf58cad418f..79bd6966f11 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -98,7 +98,7 @@ $ qsub -I ``` After some time, a shell will be started on the `skitty` cluster. -You can now create a virtual environment as described in [this section](#creating-a-venv). +You can now create a virtual environment as described in [the first section](#creating-a-venv). This virtual environment can be used by jobs running on the `skitty` cluster. ## Using virtual environments in job scripts From c41b79c761e3f076401092ee5cff611c55337bd4 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 16 Jul 2024 16:09:43 +0200 Subject: [PATCH 06/52] moved warning --- .../setting_up_python_virtual_environments.md | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 79bd6966f11..62a1f79b8b2 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -4,6 +4,8 @@ Python virtual environments (venv's) are a way to create isolated Python environments. You can install packages in a venv without affecting the system Python installation. +Because a normal user cannot install packages globally, +venv's are a way to install packages without needing root access. This is useful when you need to use a package that is not available as a module on the HPC cluster. ## Creating a venv @@ -64,6 +66,21 @@ While in the virtual environment, we can run the script without any issues: $ python pytorch_poutine_example.py ``` + +## Creating a virtual environment on a specific cluster + +To create a virtual environment for a specific cluster, you need to start an interactive shell on that cluster. +Let's say you want to create a virtual environment on the `skitty` cluster. + +```bash +$ module swap cluster/skitty +$ qsub -I +``` + +After some time, a shell will be started on the `skitty` cluster. +You can now create a virtual environment as described in [the first section](#creating-a-venv). +This virtual environment can be used by jobs running on the `skitty` cluster. + !!! warning Activating a virtual environment (venv) created on a different cluster can cause issues. This happens because the binaries in the venv from cluster A might not work with the CPU architecture of cluster B. @@ -87,20 +104,6 @@ $ python pytorch_poutine_example.py we are presented with the illegal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) -## Creating a virtual environment on a specific cluster - -To create a virtual environment for a specific cluster, you need to start an interactive shell on that cluster. -Let's say you want to create a virtual environment on the `skitty` cluster. - -```bash -$ module swap cluster/skitty -$ qsub -I -``` - -After some time, a shell will be started on the `skitty` cluster. -You can now create a virtual environment as described in [the first section](#creating-a-venv). -This virtual environment can be used by jobs running on the `skitty` cluster. - ## Using virtual environments in job scripts After creating a virtual environment for a cluster, you need to activate it in your job script. From 642ba33f465faf4d9578e1bf92e256f1166030be Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 16 Jul 2024 17:12:27 +0200 Subject: [PATCH 07/52] added example python job --- .../setting_up_python_virtual_environments.md | 66 +++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 62a1f79b8b2..1ac2deeda6f 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -44,7 +44,7 @@ $ deactivate You can combine packages installed in a venv with modules. The following script uses pytorch (which is available as a module) and Poutyne (which is not available as a module): -```python title="pytorch_poutine_example.py" +```python title="pytorch_poutine.py" import torch import poutyne @@ -63,7 +63,7 @@ $ pip install Poutyne While in the virtual environment, we can run the script without any issues: ```bash -$ python pytorch_poutine_example.py +$ python pytorch_poutine.py ``` @@ -104,28 +104,68 @@ This virtual environment can be used by jobs running on the `skitty` cluster. we are presented with the illegal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) -## Using virtual environments in job scripts +## Example Python job -After creating a virtual environment for a cluster, you need to activate it in your job script. +This section will combine the concepts discussed in the previous sections to: + +1. Create a virtual environment on a specific cluster. +2. Combine packages installed in the venv with modules. +3. Submit a job script that uses the virtual environment. + +The example script that we will run is the following: + +```python title="pytorch_poutine.py" +import torch +import poutyne + +print(f"The version of PyTorch is: {torch.__version__}") +print(f"The version of Poutine is: {poutyne.__version__}") +``` + +First, we create a virtual environment on the `skitty` cluster: + +```bash +$ module swap cluster/skitty +$ qsub -I +# Load module dependencies +$ ml PyTorch/2.1.2-foss-2023a +$ python -m venv myenv +$ source myenv/bin/activate +# install virtual environment dependencies +$ pip install Poutyne +$ deactivate +``` + +We exit the interactive shell by pressing `CTRL+D` and create a job script that loads the PyTorch module, +enters the venv and executes the script: ```bash title="jobscript.pbs" #!/bin/bash # Basic parameters -#PBS -N jobname ## Job name -#PBS -l nodes=1:ppn=2 ## 1 node, 2 processors per node (ppn=all to get a full node) -#PBS -l walltime=01:00:00 ## Max time your job will run (no more than 72:00:00) +#PBS -N python_job_example ## Job name +#PBS -l nodes=1:ppn=1 ## 1 node, 1 processors per node +#PBS -l walltime=01:00:00 ## Max time your job will run (no more than 72:00:00) -module load [module] -module load [module] +ml PyTorch/2.1.2-foss-2023a # Load the PyTorch module -cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted +cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted -source myenv/bin/activate # Activate the virtual environment -python myscript.py # Run your Python script, or any other command within the virtual environment -deactivate # Deactivate the virtual environment +source myenv/bin/activate # Activate the virtual environment +python pytorch_poutine.py # Run your Python script, or any other command within the virtual environment +deactivate # Deactivate the virtual environment ``` +Next, we submit the job script: + +```bash +$ qsub jobscript.pbs +``` + +Two files will be created in the directory where the job was submitted: `python_job_example.o[job_id]` and `python_job_example.e[job_id]`. +The `.o` file contains the output of the job. + + ## Troubleshooting ### Error: cannot open shared object file: No such file or directory From b01aedc3b77f1d00c4a3df544f66fa67bbf938fb Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Fri, 19 Jul 2024 17:00:07 +0200 Subject: [PATCH 08/52] skitty -> shinx --- .../HPC/setting_up_python_virtual_environments.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 1ac2deeda6f..fd44e729696 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -70,16 +70,16 @@ $ python pytorch_poutine.py ## Creating a virtual environment on a specific cluster To create a virtual environment for a specific cluster, you need to start an interactive shell on that cluster. -Let's say you want to create a virtual environment on the `skitty` cluster. +Let's say you want to create a virtual environment on the `shinx` cluster. ```bash -$ module swap cluster/skitty +$ module swap cluster/shinx $ qsub -I ``` -After some time, a shell will be started on the `skitty` cluster. +After some time, a shell will be started on the `shinx` cluster. You can now create a virtual environment as described in [the first section](#creating-a-venv). -This virtual environment can be used by jobs running on the `skitty` cluster. +This virtual environment can be used by jobs running on the `shinx` cluster. !!! warning Activating a virtual environment (venv) created on a different cluster can cause issues. @@ -122,10 +122,10 @@ print(f"The version of PyTorch is: {torch.__version__}") print(f"The version of Poutine is: {poutyne.__version__}") ``` -First, we create a virtual environment on the `skitty` cluster: +First, we create a virtual environment on the `shinx` cluster: ```bash -$ module swap cluster/skitty +$ module swap cluster/shinx $ qsub -I # Load module dependencies $ ml PyTorch/2.1.2-foss-2023a From 7c120500f0018381d9c0ba184763cd70235c08d9 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 12:32:18 +0200 Subject: [PATCH 09/52] typo: poutine -> poutyne --- .../docs/HPC/setting_up_python_virtual_environments.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index fd44e729696..fc98f2ac2eb 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -44,7 +44,7 @@ $ deactivate You can combine packages installed in a venv with modules. The following script uses pytorch (which is available as a module) and Poutyne (which is not available as a module): -```python title="pytorch_poutine.py" +```python title="pytorch_poutyne.py" import torch import poutyne @@ -63,7 +63,7 @@ $ pip install Poutyne While in the virtual environment, we can run the script without any issues: ```bash -$ python pytorch_poutine.py +$ python pytorch_poutyne.py ``` @@ -114,12 +114,12 @@ This section will combine the concepts discussed in the previous sections to: The example script that we will run is the following: -```python title="pytorch_poutine.py" +```python title="pytorch_poutyne.py" import torch import poutyne print(f"The version of PyTorch is: {torch.__version__}") -print(f"The version of Poutine is: {poutyne.__version__}") +print(f"The version of Poutyne is: {poutyne.__version__}") ``` First, we create a virtual environment on the `shinx` cluster: @@ -152,7 +152,7 @@ ml PyTorch/2.1.2-foss-2023a # Load the PyTorch module cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted source myenv/bin/activate # Activate the virtual environment -python pytorch_poutine.py # Run your Python script, or any other command within the virtual environment +python pytorch_poutyne.py # Run your Python script, or any other command within the virtual environment deactivate # Deactivate the virtual environment ``` From ae81522bf2521ad2247ecdc6186f50b4931ed127 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:11:21 +0200 Subject: [PATCH 10/52] Rephrase sentence about python virtual environments Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index fc98f2ac2eb..3ab80ff255d 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -2,7 +2,7 @@ ## Introduction -Python virtual environments (venv's) are a way to create isolated Python environments. +A Python virtual environment (`venv` for short) is a way to create an isolated Python environment. You can install packages in a venv without affecting the system Python installation. Because a normal user cannot install packages globally, venv's are a way to install packages without needing root access. From 08012dfae55b2729acd000c21da0c3f9c60070ec Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:11:55 +0200 Subject: [PATCH 11/52] install packages -> install additional packages Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 3ab80ff255d..011bc4cf49a 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -3,7 +3,7 @@ ## Introduction A Python virtual environment (`venv` for short) is a way to create an isolated Python environment. -You can install packages in a venv without affecting the system Python installation. +You can install additional Python packages in a virtual environment without affecting the system Python installation. Because a normal user cannot install packages globally, venv's are a way to install packages without needing root access. This is useful when you need to use a package that is not available as a module on the HPC cluster. From 6fd0b291a0ae56ea64cac384c64e3450d966756a Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:12:44 +0200 Subject: [PATCH 12/52] venv -> python virtual environment Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 011bc4cf49a..ce40f3473f0 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -10,7 +10,7 @@ This is useful when you need to use a package that is not available as a module ## Creating a venv -A venv can be created with the following commands: +A Python virtual environment can be created with the following commands: ```bash $ python -m venv myenv # Create a new venv named myenv From bdf7be77e415de2abcb30436f7a2cf77f12f6c9b Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:12:58 +0200 Subject: [PATCH 13/52] Update mkdocs/docs/HPC/setting_up_python_virtual_environments.md Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index ce40f3473f0..0c0009c2a2d 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -8,7 +8,7 @@ Because a normal user cannot install packages globally, venv's are a way to install packages without needing root access. This is useful when you need to use a package that is not available as a module on the HPC cluster. -## Creating a venv +## Creating a Python virtual environment A Python virtual environment can be created with the following commands: From d337b6d4c075165482d9c80a9f190d54e5864a1b Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:14:14 +0200 Subject: [PATCH 14/52] clarify which modules Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 0c0009c2a2d..ff6f0ed57e2 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -31,7 +31,7 @@ $ deactivate ``` !!! note - Always prefer to use modules if they are available. + Always prefer to use Python packages that are centrally installed, which are available via the environment modules interface. Modules are compiled and optimized for the HPC cluster, while packages installed with `pip` are not. To check if a package is available as a module, you can use the following command: From 87a30afc16070d39726aa37139fbdc62b1162432 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 13:32:01 +0200 Subject: [PATCH 15/52] rewrite introduction to better reflect python environmens --- .../HPC/setting_up_python_virtual_environments.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index ff6f0ed57e2..6f0b5c139fb 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -2,11 +2,13 @@ ## Introduction -A Python virtual environment (`venv` for short) is a way to create an isolated Python environment. -You can install additional Python packages in a virtual environment without affecting the system Python installation. -Because a normal user cannot install packages globally, -venv's are a way to install packages without needing root access. -This is useful when you need to use a package that is not available as a module on the HPC cluster. +A Python virtual environment (venv for short) +is a tool to create an isolated Python workspace with its own dependencies. +Within this isolated environment, +you can install additional Python packages without affecting the system-wide Python installation. +Because a normal user cannot install packages globally, +using a virtual environment allows you to install packages locally without needing administrator privileges. +This is especially useful when you need to use a package that is not available as a module on the HPC cluster. ## Creating a Python virtual environment From 33ac8a20833fe1dcca543de078d03d72a4a89e7f Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 13:38:27 +0200 Subject: [PATCH 16/52] venv -> virtual environment --- .../setting_up_python_virtual_environments.md | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 6f0b5c139fb..ef9dcfc8a13 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -15,18 +15,19 @@ This is especially useful when you need to use a package that is not available a A Python virtual environment can be created with the following commands: ```bash -$ python -m venv myenv # Create a new venv named myenv -$ source myenv/bin/activate # Activate the venv +$ python -m venv myenv # Create a new virtual environment named myenv +$ source myenv/bin/activate # Activate the virtual environment ``` -After activating the venv, you can install packages with `pip`: +After activating the virtual environment, you can install packages with `pip`: ```bash $ pip install example_package1 $ pip install example_package2 ``` -It is now possible to run Python scripts that use the installed packages in the venv. To deactivate the venv, run: +It is now possible to run Python scripts that use the installed packages in the virtual environment. +To deactivate it, run: ```bash $ deactivate @@ -41,9 +42,9 @@ $ deactivate $ module av package_name ``` -## Combining venv's with modules +## Combining virtual environments with modules -You can combine packages installed in a venv with modules. The following script uses +You can combine packages installed in a virtual environment with modules. The following script uses pytorch (which is available as a module) and Poutyne (which is not available as a module): ```python title="pytorch_poutyne.py" @@ -53,7 +54,7 @@ import poutyne ... ``` -We load a PyTorch package as a module and install Poutyne in a venv: +We load a PyTorch package as a module and install Poutyne in a virtual environment: ```bash $ ml PyTorch/2.1.2-foss-2023a @@ -84,10 +85,10 @@ You can now create a virtual environment as described in [the first section](#cr This virtual environment can be used by jobs running on the `shinx` cluster. !!! warning - Activating a virtual environment (venv) created on a different cluster can cause issues. - This happens because the binaries in the venv from cluster A might not work with the CPU architecture of cluster B. + Activating a virtual environment created on a different cluster can cause issues. + This happens because the binaries in the virtual environments from cluster A might not work with the CPU architecture of cluster B. - For example, if we create a venv on the skitty cluster, + For example, if we create a virtual environment on the skitty cluster, ```bash $ module swap cluster/skitty @@ -111,7 +112,7 @@ This virtual environment can be used by jobs running on the `shinx` cluster. This section will combine the concepts discussed in the previous sections to: 1. Create a virtual environment on a specific cluster. -2. Combine packages installed in the venv with modules. +2. Combine packages installed in the virtual environment with modules. 3. Submit a job script that uses the virtual environment. The example script that we will run is the following: @@ -139,7 +140,7 @@ $ deactivate ``` We exit the interactive shell by pressing `CTRL+D` and create a job script that loads the PyTorch module, -enters the venv and executes the script: +enters the virtual environment and executes the script: ```bash title="jobscript.pbs" #!/bin/bash From 4f1a65d3fe7341745ac8b3844705d58f3e78f6d6 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 13:53:11 +0200 Subject: [PATCH 17/52] qualify packages and modules --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index ef9dcfc8a13..bea2645e8f0 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -44,8 +44,8 @@ $ deactivate ## Combining virtual environments with modules -You can combine packages installed in a virtual environment with modules. The following script uses -pytorch (which is available as a module) and Poutyne (which is not available as a module): +You can combine Python packages installed in a virtual environment with environment modules. +The following script uses pytorch (which is available as a module) and Poutyne (which is not available as a module): ```python title="pytorch_poutyne.py" import torch From 0f603efaa8cda5010e587b7f49b43be61478fb0b Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 13:55:55 +0200 Subject: [PATCH 18/52] pytorch -> PyTorch + assume Poutyne not centrally installed --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index bea2645e8f0..a11017980e2 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -45,7 +45,8 @@ $ deactivate ## Combining virtual environments with modules You can combine Python packages installed in a virtual environment with environment modules. -The following script uses pytorch (which is available as a module) and Poutyne (which is not available as a module): +The following script uses PyTorch (which is available as a module) +and Poutyne (which we assume is not centrally installed): ```python title="pytorch_poutyne.py" import torch From 411c9d69a6b72d0f5b3b656d4a0cc2fc14e462d5 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 13:57:23 +0200 Subject: [PATCH 19/52] ml -> module load --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index a11017980e2..3e6d239f00b 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -58,7 +58,7 @@ import poutyne We load a PyTorch package as a module and install Poutyne in a virtual environment: ```bash -$ ml PyTorch/2.1.2-foss-2023a +$ module load PyTorch/2.1.2-foss-2023a $ python -m venv myenv $ source myenv/bin/activate $ pip install Poutyne @@ -132,7 +132,7 @@ First, we create a virtual environment on the `shinx` cluster: $ module swap cluster/shinx $ qsub -I # Load module dependencies -$ ml PyTorch/2.1.2-foss-2023a +$ module load PyTorch/2.1.2-foss-2023a $ python -m venv myenv $ source myenv/bin/activate # install virtual environment dependencies @@ -151,7 +151,7 @@ enters the virtual environment and executes the script: #PBS -l nodes=1:ppn=1 ## 1 node, 1 processors per node #PBS -l walltime=01:00:00 ## Max time your job will run (no more than 72:00:00) -ml PyTorch/2.1.2-foss-2023a # Load the PyTorch module +module load PyTorch/2.1.2-foss-2023a # Load the PyTorch module cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted From 24c6c33f185f2f62abf9c1a3f121aade5fec2dfa Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 14:38:48 +0200 Subject: [PATCH 20/52] change 'when in a virtual environment' to 'when a virtual environment is activated' --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 3e6d239f00b..365dceb947d 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -64,7 +64,7 @@ $ source myenv/bin/activate $ pip install Poutyne ``` -While in the virtual environment, we can run the script without any issues: +While the virtual environment is activated, we can run the script without any issues: ```bash $ python pytorch_poutyne.py @@ -177,7 +177,7 @@ The `.o` file contains the output of the job. There are two main reasons why this error could occur. 1. you have not loaded the python module that was used to create the virtual environment. -2. you added or removed modules while in the virtual environment. +2. you added or removed modules while the virtual environment was activated. #### Entering a virtual environment while the python module used to create it is not active From 6754ddd8252d1742c4f4686ddf13bbb32cb2d3db Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 14:40:38 +0200 Subject: [PATCH 21/52] on a specific cluster -> for a specific cluster' --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 365dceb947d..74f11126865 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -71,7 +71,7 @@ $ python pytorch_poutyne.py ``` -## Creating a virtual environment on a specific cluster +## Creating a virtual environment for a specific cluster To create a virtual environment for a specific cluster, you need to start an interactive shell on that cluster. Let's say you want to create a virtual environment on the `shinx` cluster. From f623b0b4e524a0c706f69dba2a50d7b2e0264a4c Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 15:23:19 +0200 Subject: [PATCH 22/52] creating, activating, using and deactivating are now subsections --- .../setting_up_python_virtual_environments.md | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 74f11126865..8a6ab56e8d1 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -10,15 +10,32 @@ Because a normal user cannot install packages globally, using a virtual environment allows you to install packages locally without needing administrator privileges. This is especially useful when you need to use a package that is not available as a module on the HPC cluster. -## Creating a Python virtual environment +## Managing Python Environments -A Python virtual environment can be created with the following commands: +This section will explain how to create, activate, use and deactivate Python virtual environments. + +### Creating a Python virtual environment + +A Python virtual environment can be created with the following command: ```bash $ python -m venv myenv # Create a new virtual environment named myenv +``` + +This command creates a new directory named `myenv` in the current working directory. +This directory will contain the packages, scripts, and binaries that are needed to manage the virtual environment. + +### Activating a virtual environment + +To use the virtual environment, you need to activate it. +This will modify the shell environment to use the Python interpreter and packages from the virtual environment. + +```bash $ source myenv/bin/activate # Activate the virtual environment ``` +### Installing packages in a virtual environment + After activating the virtual environment, you can install packages with `pip`: ```bash @@ -26,12 +43,11 @@ $ pip install example_package1 $ pip install example_package2 ``` -It is now possible to run Python scripts that use the installed packages in the virtual environment. -To deactivate it, run: +These packages will be scoped to the virtual environment and will not affect the system-wide Python installation, +and are only available when the virtual environment is activated. +No administrator privileges are required to install packages in a virtual environment. -```bash -$ deactivate -``` +It is now possible to run Python scripts that use the installed packages in the virtual environment. !!! note Always prefer to use Python packages that are centrally installed, which are available via the environment modules interface. @@ -42,7 +58,17 @@ $ deactivate $ module av package_name ``` -## Combining virtual environments with modules + +### Deactivating a virtual environment + +When you are done using the virtual environment, you can deactivate it. +To do that, run: + +```bash +$ deactivate +``` + +## Combining virtual environments with centrally installed modules You can combine Python packages installed in a virtual environment with environment modules. The following script uses PyTorch (which is available as a module) From b80340f01448eb04d010a44d05285b28fece3cc4 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 17:22:55 +0200 Subject: [PATCH 23/52] add info about bundle modules --- .../docs/HPC/setting_up_python_virtual_environments.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 8a6ab56e8d1..d79003a8331 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -58,6 +58,15 @@ It is now possible to run Python scripts that use the installed packages in the $ module av package_name ``` + Some Python packages are installed as extensions of modules. For example, `numpy`, `scipy` and `pandas` are + part of the `SciPy-bundle` module. You can use + + ```bash + $ module show + ``` + + to check which extensions are included in a module, if any. + ### Deactivating a virtual environment From 2ae0ad7fe7329635235f2bbf3847243ae5278857 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 17:28:20 +0200 Subject: [PATCH 24/52] Moved warning about activating venv made on other cluster to its own section --- .../setting_up_python_virtual_environments.md | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index d79003a8331..44e4fee4a6c 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -120,28 +120,6 @@ After some time, a shell will be started on the `shinx` cluster. You can now create a virtual environment as described in [the first section](#creating-a-venv). This virtual environment can be used by jobs running on the `shinx` cluster. -!!! warning - Activating a virtual environment created on a different cluster can cause issues. - This happens because the binaries in the virtual environments from cluster A might not work with the CPU architecture of cluster B. - - For example, if we create a virtual environment on the skitty cluster, - - ```bash - $ module swap cluster/skitty - $ qsub -I - $ python -m venv myenv - ``` - - return to the login node by pressing CTRL+D and try to use the virtual environment: - - ```bash - $ source myenv/bin/activate - $ python - Illegal instruction (core dumped) - ``` - - we are presented with the illegal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) - ## Example Python job @@ -207,6 +185,30 @@ The `.o` file contains the output of the job. ## Troubleshooting +### Illegal instruction error + +Activating a virtual environment created on a different cluster can cause issues. +This happens +because the binaries in the virtual environments from cluster A might not work with the CPU architecture of cluster B. + +For example, if we create a virtual environment on the skitty cluster, + +```bash +$ module swap cluster/skitty +$ qsub -I +$ python -m venv myenv +``` + +return to the login node by pressing CTRL+D and try to use the virtual environment: + +```bash +$ source myenv/bin/activate +$ python +Illegal instruction (core dumped) +``` + +we are presented with the illegal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) + ### Error: cannot open shared object file: No such file or directory There are two main reasons why this error could occur. From 193434d86c431daf001d4935da380d241dfe90d3 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Mon, 19 Aug 2024 17:32:18 +0200 Subject: [PATCH 25/52] shinx -> donphan --- .../HPC/setting_up_python_virtual_environments.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 44e4fee4a6c..b2028043294 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -109,16 +109,16 @@ $ python pytorch_poutyne.py ## Creating a virtual environment for a specific cluster To create a virtual environment for a specific cluster, you need to start an interactive shell on that cluster. -Let's say you want to create a virtual environment on the `shinx` cluster. +Let's say you want to create a virtual environment on the `donphan` cluster. ```bash -$ module swap cluster/shinx +$ module swap cluster/donphan $ qsub -I ``` -After some time, a shell will be started on the `shinx` cluster. +After some time, a shell will be started on the `donphan` cluster. You can now create a virtual environment as described in [the first section](#creating-a-venv). -This virtual environment can be used by jobs running on the `shinx` cluster. +This virtual environment can be used by jobs running on the `donphan` cluster. ## Example Python job @@ -139,10 +139,10 @@ print(f"The version of PyTorch is: {torch.__version__}") print(f"The version of Poutyne is: {poutyne.__version__}") ``` -First, we create a virtual environment on the `shinx` cluster: +First, we create a virtual environment on the `donphan` cluster: ```bash -$ module swap cluster/shinx +$ module swap cluster/donphan $ qsub -I # Load module dependencies $ module load PyTorch/2.1.2-foss-2023a From bc9370742c05cb7baaf799e151cfa0b06b2e8ccd Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:23:33 +0200 Subject: [PATCH 26/52] quotes around venv Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index b2028043294..bf12fa2eacd 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -2,7 +2,7 @@ ## Introduction -A Python virtual environment (venv for short) +A Python virtual environment ("venv" for short) is a tool to create an isolated Python workspace with its own dependencies. Within this isolated environment, you can install additional Python packages without affecting the system-wide Python installation. From 2bc34af88ea4cd0bbcecaf9b5f6faa373cd0396b Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 14:26:24 +0200 Subject: [PATCH 27/52] removed 'dependencies' from sentence --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index bf12fa2eacd..85dec952217 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -3,7 +3,7 @@ ## Introduction A Python virtual environment ("venv" for short) -is a tool to create an isolated Python workspace with its own dependencies. +is a tool to create an isolated Python workspace. Within this isolated environment, you can install additional Python packages without affecting the system-wide Python installation. Because a normal user cannot install packages globally, From 362d3241211ec168cab3c73a21fa2b533a7251dd Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:28:30 +0200 Subject: [PATCH 28/52] quotes around 'myenv' Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 85dec952217..41fd10df1d5 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -19,7 +19,7 @@ This section will explain how to create, activate, use and deactivate Python vir A Python virtual environment can be created with the following command: ```bash -$ python -m venv myenv # Create a new virtual environment named myenv +$ python -m venv myenv # Create a new virtual environment named 'myenv' ``` This command creates a new directory named `myenv` in the current working directory. From 547cdfd03f8dc73507927498b627a431fea8136a Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:29:37 +0200 Subject: [PATCH 29/52] directory -> subdirectory Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 41fd10df1d5..bcd8347f890 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -22,7 +22,7 @@ A Python virtual environment can be created with the following command: $ python -m venv myenv # Create a new virtual environment named 'myenv' ``` -This command creates a new directory named `myenv` in the current working directory. +This command creates a new subdirectory named `myenv` in the current working directory. This directory will contain the packages, scripts, and binaries that are needed to manage the virtual environment. ### Activating a virtual environment From 4b1c79e675778894c7af8b2ef03b45987bbd75b1 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:30:08 +0200 Subject: [PATCH 30/52] added spacing before comment Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index bcd8347f890..6997ea8e935 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -31,7 +31,7 @@ To use the virtual environment, you need to activate it. This will modify the shell environment to use the Python interpreter and packages from the virtual environment. ```bash -$ source myenv/bin/activate # Activate the virtual environment +$ source myenv/bin/activate # Activate the virtual environment ``` ### Installing packages in a virtual environment From 3431158a22e2d94d415369394cd892beeb033d32 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:30:28 +0200 Subject: [PATCH 31/52] Capitalize 'python' Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 6997ea8e935..480b28e9661 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -216,7 +216,7 @@ There are two main reasons why this error could occur. 1. you have not loaded the python module that was used to create the virtual environment. 2. you added or removed modules while the virtual environment was activated. -#### Entering a virtual environment while the python module used to create it is not active +#### Entering a virtual environment while the Python module used to create it is not active When you load a python module and use that to make a virtual environment, you need to make sure that the same module is loaded when you enter the environment. This is because the virtual environment keeps a reference to the base python From 045ca81dbf7719285a49a0615da95f27703402d3 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:30:57 +0200 Subject: [PATCH 32/52] when sentence -> if sentence Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 480b28e9661..94ca335599f 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -218,7 +218,7 @@ There are two main reasons why this error could occur. #### Entering a virtual environment while the Python module used to create it is not active -When you load a python module and use that to make a virtual environment, you need to make sure that the same module +If you loaded a `Python` module when creating a virtual environment, you need to make sure that the same module is loaded when you enter the environment. This is because the virtual environment keeps a reference to the base python used to create it. From 1a2f3f1a2cb4041e239bbaad9a3b454d6b084c13 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 14:35:15 +0200 Subject: [PATCH 33/52] clarify wrong command in example --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 94ca335599f..9cbdf2dead2 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -227,13 +227,14 @@ The following commands illustrate this issue: ```bash $ module load Python/3.10.8-GCCcore-12.2.0 # Load a python module $ python -m venv myenv # Create a virtual environment with loaded python module -$ module purge # Remove all loaded modules +$ module purge # Remove all loaded modules (WRONG!) $ source myenv/bin/activate # Activate the virtual environment $ python # Start python python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory ``` -Here, the virtual environment tries to use the python module that was loaded when the environment was created, which is no longer available. +Here, the virtual environment tries to use the python module that was loaded when the environment was created. +Since we used `module purge`, that module is no longer available. The solution is to load the same python module before activating the virtual environment: ```bash From 1b329116a44f829dda6fca85cac40c4482a93335 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:36:28 +0200 Subject: [PATCH 34/52] Italic 'activate' Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 9cbdf2dead2..c58002305a7 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -27,7 +27,7 @@ This directory will contain the packages, scripts, and binaries that are needed ### Activating a virtual environment -To use the virtual environment, you need to activate it. +To use the virtual environment, you need to *activate* it. This will modify the shell environment to use the Python interpreter and packages from the virtual environment. ```bash From abaf6a682254a5e33f61fcd6e3affbd78351b1eb Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:36:53 +0200 Subject: [PATCH 35/52] Uppercase modifying Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index c58002305a7..2918d67e823 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -242,7 +242,7 @@ $ module load Python/3.10.8-GCCcore-12.2.0 # Load the same python module $ source myenv/bin/activate # Activate the virtual environment ``` -#### modifying modules while in a virtual environment +#### Modifying modules while in a virtual environment You must not delete or add modules while in a virtual environment. Adding and removing modules modifies the `$PATH` variable in the current shell. When activating a virtual environment, From d918ca271a24fc814e026916c0874b43b2cf7074 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:37:53 +0200 Subject: [PATCH 36/52] 'packages' -> 'additional packages' and 'pip' -> 'pip install' Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 2918d67e823..62efff083b2 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -36,7 +36,7 @@ $ source myenv/bin/activate # Activate the virtual environmen ### Installing packages in a virtual environment -After activating the virtual environment, you can install packages with `pip`: +After activating the virtual environment, you can install additional Python packages with `pip install`: ```bash $ pip install example_package1 From b9f25feeeb7cb1871154c5cd686eb148f4c00761 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:38:52 +0200 Subject: [PATCH 37/52] replace 'modules' with more descriptive sentence Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 62efff083b2..7a2fb3ea9de 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -51,7 +51,7 @@ It is now possible to run Python scripts that use the installed packages in the !!! note Always prefer to use Python packages that are centrally installed, which are available via the environment modules interface. - Modules are compiled and optimized for the HPC cluster, while packages installed with `pip` are not. + Software installations available via the modules interface are compiled and optimized for the HPC cluster, while packages installed with `pip` are not. To check if a package is available as a module, you can use the following command: ```bash From 00ebe03f1fc3b3fe3e2015b447e853c7f7840f3e Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:39:23 +0200 Subject: [PATCH 38/52] ', if any' -> '(if any)' Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 7a2fb3ea9de..9de552876f3 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -65,7 +65,7 @@ It is now possible to run Python scripts that use the installed packages in the $ module show ``` - to check which extensions are included in a module, if any. + to check which extensions are included in a module (if any). ### Deactivating a virtual environment From 6fa2f76d3cc850a43bd5f8c9d8462d0b4f9c5a5e Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 14:43:14 +0200 Subject: [PATCH 39/52] consistent module wording. 'add' -> 'load' and 'remove', 'delete' -> 'unload' --- .../HPC/setting_up_python_virtual_environments.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 9de552876f3..93bcbc4537d 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -214,7 +214,7 @@ we are presented with the illegal instruction error. More info on this [here](tr There are two main reasons why this error could occur. 1. you have not loaded the python module that was used to create the virtual environment. -2. you added or removed modules while the virtual environment was activated. +2. you loaded or unloaded modules while the virtual environment was activated. #### Entering a virtual environment while the Python module used to create it is not active @@ -227,7 +227,7 @@ The following commands illustrate this issue: ```bash $ module load Python/3.10.8-GCCcore-12.2.0 # Load a python module $ python -m venv myenv # Create a virtual environment with loaded python module -$ module purge # Remove all loaded modules (WRONG!) +$ module purge # unload all modules (WRONG!) $ source myenv/bin/activate # Activate the virtual environment $ python # Start python python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory @@ -244,11 +244,11 @@ $ source myenv/bin/activate # Activate the virtual environment #### Modifying modules while in a virtual environment -You must not delete or add modules while in a virtual environment. -Adding and removing modules modifies the `$PATH` variable in the current shell. When activating a virtual environment, -it will store the `$PATH` variable of the shell at that moment. If you modify the `$PATH` variable while in a virtual environment by loading or deleting modules, +You must not load or unload modules while in a virtual environment. +Loading and unloading modules modifies the `$PATH` variable in the current shell. When activating a virtual environment, +it will store the `$PATH` variable of the shell at that moment. If you modify the `$PATH` variable while in a virtual environment by loading or unloading modules, and deactivate the virtual environment, the `$PATH` variable will be reset to the one stored in the virtual environment. -trying to use those modules will lead to errors: +Trying to use those modules will lead to errors: ```bash $ module load Python/3.10.8-GCCcore-12.2.0 # Load a python module From 66cf0bf20941d392b3c7bc06aa3e474a8d1dd99a Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 14:47:13 +0200 Subject: [PATCH 40/52] add deactivate to the end of the Poutyne example --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 93bcbc4537d..d8c9a95a165 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -105,6 +105,12 @@ While the virtual environment is activated, we can run the script without any is $ python pytorch_poutyne.py ``` +Deactivate the virtual environment when you are done: + +```bash +$ deactivate +``` + ## Creating a virtual environment for a specific cluster From 598ce4f5bfe0dcb3bc3b41e307ce93eaa40ae267 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 14:52:45 +0200 Subject: [PATCH 41/52] 'exit' instead of CTRL+D to exit the interactive shell --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index d8c9a95a165..0c44ce606d6 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -159,8 +159,8 @@ $ pip install Poutyne $ deactivate ``` -We exit the interactive shell by pressing `CTRL+D` and create a job script that loads the PyTorch module, -enters the virtual environment and executes the script: +Type `exit` to exit the interactive shell. +We now create a job script that loads the PyTorch module, enters the virtual environment and executes the script: ```bash title="jobscript.pbs" #!/bin/bash From 9931d45fde92a052980b74de79ecb1271de1656d Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 15:00:51 +0200 Subject: [PATCH 42/52] source line is now below module load --- .../setting_up_python_virtual_environments.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 0c44ce606d6..5268f0595ad 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -166,17 +166,16 @@ We now create a job script that loads the PyTorch module, enters the virtual env #!/bin/bash # Basic parameters -#PBS -N python_job_example ## Job name -#PBS -l nodes=1:ppn=1 ## 1 node, 1 processors per node -#PBS -l walltime=01:00:00 ## Max time your job will run (no more than 72:00:00) +#PBS -N python_job_example ## Job name +#PBS -l nodes=1:ppn=1 ## 1 node, 1 processors per node +#PBS -l walltime=01:00:00 ## Max time your job will run (no more than 72:00:00) -module load PyTorch/2.1.2-foss-2023a # Load the PyTorch module +module load PyTorch/2.1.2-foss-2023a # Load the PyTorch module +cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted +source myenv/bin/activate # Activate the virtual environment -cd $PBS_O_WORKDIR # Change working directory to the location where the job was submitted - -source myenv/bin/activate # Activate the virtual environment -python pytorch_poutyne.py # Run your Python script, or any other command within the virtual environment -deactivate # Deactivate the virtual environment +python pytorch_poutyne.py # Run your Python script, or any other command within the virtual environment +deactivate # Deactivate the virtual environment ``` Next, we submit the job script: From a5001ee3343dd4090de5a98faa9b35bf48a41258 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 15:13:26 +0200 Subject: [PATCH 43/52] add blank line above deactivate --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 5268f0595ad..a717c836f16 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -175,6 +175,7 @@ cd $PBS_O_WORKDIR # Change working directory to the location source myenv/bin/activate # Activate the virtual environment python pytorch_poutyne.py # Run your Python script, or any other command within the virtual environment + deactivate # Deactivate the virtual environment ``` From a833b02430c595ff47dac252f8261a025347c608 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 15:19:15 +0200 Subject: [PATCH 44/52] use {{jobid}} placeholder --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index a717c836f16..1680b184a43 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -185,7 +185,8 @@ Next, we submit the job script: $ qsub jobscript.pbs ``` -Two files will be created in the directory where the job was submitted: `python_job_example.o[job_id]` and `python_job_example.e[job_id]`. +Two files will be created in the directory where the job was submitted: +`python_job_example.o{{jobid}}` and `python_job_example.e{{job_id}}`, where {{jobid}} is the id of your job. The `.o` file contains the output of the job. From 9be32e0e8b505ac7d1b14f3c674eaf38a1d0913d Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres <63658577+lbarraga@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:22:57 +0200 Subject: [PATCH 45/52] Capitalize 'you' and 'python' Co-authored-by: Kenneth Hoste --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 1680b184a43..4aafd43f8f9 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -220,7 +220,7 @@ we are presented with the illegal instruction error. More info on this [here](tr There are two main reasons why this error could occur. -1. you have not loaded the python module that was used to create the virtual environment. +1. You have not loaded the `Python` module that was used to create the virtual environment. 2. you loaded or unloaded modules while the virtual environment was activated. #### Entering a virtual environment while the Python module used to create it is not active From 27d4a3f0c0409a6e3e1128f8427a6c52cac44f17 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 15:24:36 +0200 Subject: [PATCH 46/52] Capitalize sentence --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 4aafd43f8f9..1ca59740733 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -221,7 +221,7 @@ we are presented with the illegal instruction error. More info on this [here](tr There are two main reasons why this error could occur. 1. You have not loaded the `Python` module that was used to create the virtual environment. -2. you loaded or unloaded modules while the virtual environment was activated. +2. You loaded or unloaded modules while the virtual environment was activated. #### Entering a virtual environment while the Python module used to create it is not active From 6bd51a70e7e2a66bb3c8e679f52b3c532194f3b3 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 15:39:08 +0200 Subject: [PATCH 47/52] add note introducing $VSC_INSTITUTE_CLUSTER --- mkdocs/docs/HPC/setting_up_python_virtual_environments.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 1ca59740733..3cb8edf1c54 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -126,6 +126,14 @@ After some time, a shell will be started on the `donphan` cluster. You can now create a virtual environment as described in [the first section](#creating-a-venv). This virtual environment can be used by jobs running on the `donphan` cluster. +!!! note "Naming a virtual environment" + When naming a virtual environment, it is recommended to include the name of the cluster it was created for. + We can use the `$VSC_INSTITUTE_CLUSTER` variable to get the name of the current cluster. + + ```bash + $ python -m venv myenv_${VSC_INSTITUTE_CLUSTER} + ``` + ## Example Python job From f3b8a118035fc14809396e2c5d1e27a8d8c03fe1 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 27 Aug 2024 15:50:14 +0200 Subject: [PATCH 48/52] add section on using venv --- .../HPC/setting_up_python_virtual_environments.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 3cb8edf1c54..bf9d07edecc 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -67,6 +67,21 @@ It is now possible to run Python scripts that use the installed packages in the to check which extensions are included in a module (if any). +### Using a virtual environment + +Once the environment is activated and packages are installed, +you can run Python scripts that use the installed packages: + +```python title="example.py" +import example_package1 +import example_package2 +... +``` + +```bash +python example.py +``` + ### Deactivating a virtual environment From 999b560119911946e3f2331fbfe60dc58e1fac19 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Wed, 28 Aug 2024 10:18:31 +0200 Subject: [PATCH 49/52] add GLIBC troubleshooting section --- .../HPC/setting_up_python_virtual_environments.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index bf9d07edecc..9ee42951fae 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -239,6 +239,21 @@ Illegal instruction (core dumped) we are presented with the illegal instruction error. More info on this [here](troubleshooting.md#illegal-instruction-error) +### Error: GLIBC not found + +When running a virtual environment across clusters with different major OS versions, +you might encounter a variation of the following error: + +``` +python: /lib64/libc.so.6: version `GLIBC_2.34' not found (required by python) +``` + +Make sure you do not activate a virtual environment created on a different cluster. +For more information on how to create a virtual environment for a specific cluster, +see [Creating a virtual environment for a specific cluster](#creating-a-virtual-environment-for-a-specific-cluster). +When following these steps, make sure you do not have any modules loaded when starting the interactive job. + + ### Error: cannot open shared object file: No such file or directory There are two main reasons why this error could occur. From 732b7e8492815448143fd10fbaa4f0c0302664e5 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Wed, 28 Aug 2024 10:35:49 +0200 Subject: [PATCH 50/52] move venv page from Advanced Topics to Software Best Practices + rename --- config/templates/hpc.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/templates/hpc.template b/config/templates/hpc.template index 2d98c38ffdb..3421612fc7a 100644 --- a/config/templates/hpc.template +++ b/config/templates/hpc.template @@ -23,7 +23,6 @@ nav: - Troubleshooting: troubleshooting.md - HPC Policies: sites/hpc_policies.md - Advanced topics: - - Setting Up Python Virtual Environments: setting_up_python_virtual_environments.md - Fine-tuning Job Specifications: fine_tuning_job_specifications.md - Multi-job submission: multi_job_submission.md - Compiling and testing your software on the HPC: compiling_your_software.md @@ -47,6 +46,7 @@ nav: - More on the HPC infrastructure: linux-tutorial/hpc_infrastructure.md {%- endif %} - Software-specific Best Practices: + - Python virtual environments: setting_up_python_virtual_environments.md - AlphaFold: alphafold.md - Apptainer/Singularity: apptainer.md - EasyBuild: easybuild.md From 6f181a5513af38aaa6998d331f7197642c8afcbe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Aug 2024 11:37:22 +0200 Subject: [PATCH 51/52] keep menu items in Software-specific Best Practices sorted alphabetically --- config/templates/hpc.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/templates/hpc.template b/config/templates/hpc.template index 3421612fc7a..0bd745dbc04 100644 --- a/config/templates/hpc.template +++ b/config/templates/hpc.template @@ -46,13 +46,13 @@ nav: - More on the HPC infrastructure: linux-tutorial/hpc_infrastructure.md {%- endif %} - Software-specific Best Practices: - - Python virtual environments: setting_up_python_virtual_environments.md - AlphaFold: alphafold.md - Apptainer/Singularity: apptainer.md - EasyBuild: easybuild.md - MATLAB: MATLAB.md - mympirun: mympirun.md - OpenFOAM: openFOAM.md + - Python virtual environments: setting_up_python_virtual_environments.md - FAQ: - Frequently Asked Questions: FAQ.md - Appendices: From 5875c83a917c3ab23be8cd85047d5adb2beca9e7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 28 Aug 2024 11:40:46 +0200 Subject: [PATCH 52/52] make commands related to venv easier to copy-paste --- .../setting_up_python_virtual_environments.md | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md index 9ee42951fae..e73b59bd0fd 100644 --- a/mkdocs/docs/HPC/setting_up_python_virtual_environments.md +++ b/mkdocs/docs/HPC/setting_up_python_virtual_environments.md @@ -19,7 +19,7 @@ This section will explain how to create, activate, use and deactivate Python vir A Python virtual environment can be created with the following command: ```bash -$ python -m venv myenv # Create a new virtual environment named 'myenv' +python -m venv myenv # Create a new virtual environment named 'myenv' ``` This command creates a new subdirectory named `myenv` in the current working directory. @@ -31,7 +31,7 @@ To use the virtual environment, you need to *activate* it. This will modify the shell environment to use the Python interpreter and packages from the virtual environment. ```bash -$ source myenv/bin/activate # Activate the virtual environment +source myenv/bin/activate # Activate the virtual environment ``` ### Installing packages in a virtual environment @@ -39,8 +39,8 @@ $ source myenv/bin/activate # Activate the virtual environmen After activating the virtual environment, you can install additional Python packages with `pip install`: ```bash -$ pip install example_package1 -$ pip install example_package2 +pip install example_package1 +pip install example_package2 ``` These packages will be scoped to the virtual environment and will not affect the system-wide Python installation, @@ -55,14 +55,14 @@ It is now possible to run Python scripts that use the installed packages in the To check if a package is available as a module, you can use the following command: ```bash - $ module av package_name + module av package_name ``` Some Python packages are installed as extensions of modules. For example, `numpy`, `scipy` and `pandas` are part of the `SciPy-bundle` module. You can use ```bash - $ module show + module show module_name ``` to check which extensions are included in a module (if any). @@ -89,7 +89,7 @@ When you are done using the virtual environment, you can deactivate it. To do that, run: ```bash -$ deactivate +deactivate ``` ## Combining virtual environments with centrally installed modules @@ -108,22 +108,22 @@ import poutyne We load a PyTorch package as a module and install Poutyne in a virtual environment: ```bash -$ module load PyTorch/2.1.2-foss-2023a -$ python -m venv myenv -$ source myenv/bin/activate -$ pip install Poutyne +module load PyTorch/2.1.2-foss-2023a +python -m venv myenv +source myenv/bin/activate +pip install Poutyne ``` While the virtual environment is activated, we can run the script without any issues: ```bash -$ python pytorch_poutyne.py +python pytorch_poutyne.py ``` Deactivate the virtual environment when you are done: ```bash -$ deactivate +deactivate ``` @@ -133,8 +133,8 @@ To create a virtual environment for a specific cluster, you need to start an int Let's say you want to create a virtual environment on the `donphan` cluster. ```bash -$ module swap cluster/donphan -$ qsub -I +module swap cluster/donphan +qsub -I ``` After some time, a shell will be started on the `donphan` cluster. @@ -146,7 +146,7 @@ This virtual environment can be used by jobs running on the `donphan` cluster. We can use the `$VSC_INSTITUTE_CLUSTER` variable to get the name of the current cluster. ```bash - $ python -m venv myenv_${VSC_INSTITUTE_CLUSTER} + python -m venv myenv_${VSC_INSTITUTE_CLUSTER} ``` @@ -171,15 +171,15 @@ print(f"The version of Poutyne is: {poutyne.__version__}") First, we create a virtual environment on the `donphan` cluster: ```bash -$ module swap cluster/donphan -$ qsub -I +module swap cluster/donphan +qsub -I # Load module dependencies -$ module load PyTorch/2.1.2-foss-2023a -$ python -m venv myenv -$ source myenv/bin/activate +module load PyTorch/2.1.2-foss-2023a +python -m venv myenv +source myenv/bin/activate # install virtual environment dependencies -$ pip install Poutyne -$ deactivate +pip install Poutyne +deactivate ``` Type `exit` to exit the interactive shell. @@ -205,7 +205,7 @@ deactivate # Deactivate the virtual environment Next, we submit the job script: ```bash -$ qsub jobscript.pbs +qsub jobscript.pbs ``` Two files will be created in the directory where the job was submitted: @@ -224,8 +224,8 @@ because the binaries in the virtual environments from cluster A might not work w For example, if we create a virtual environment on the skitty cluster, ```bash -$ module swap cluster/skitty -$ qsub -I +module swap cluster/skitty +qsub -I $ python -m venv myenv ``` @@ -283,8 +283,8 @@ Since we used `module purge`, that module is no longer available. The solution is to load the same python module before activating the virtual environment: ```bash -$ module load Python/3.10.8-GCCcore-12.2.0 # Load the same python module -$ source myenv/bin/activate # Activate the virtual environment +module load Python/3.10.8-GCCcore-12.2.0 # Load the same python module +source myenv/bin/activate # Activate the virtual environment ``` #### Modifying modules while in a virtual environment @@ -305,4 +305,4 @@ $ python # PATH contains a reference to the u python: error while loading shared libraries: libpython3.10.so.1.0: cannot open shared object file: No such file or directory ``` -The solution is to only modify modules when not in a virtual environment. \ No newline at end of file +The solution is to only modify modules when not in a virtual environment.