Skip to content

Commit

Permalink
Merge pull request #161 from xylar/remove-mamba
Browse files Browse the repository at this point in the history
Remove `mamba` command and switch to Miniforge3
  • Loading branch information
xylar authored Jan 4, 2024
2 parents a43fa75 + fd2a211 commit 51b0efe
Show file tree
Hide file tree
Showing 12 changed files with 242 additions and 253 deletions.
5 changes: 1 addition & 4 deletions .github/workflows/build_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,10 @@ jobs:

- if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
name: Set up Conda Environment
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: "polaris_ci"
miniforge-variant: Mambaforge
miniforge-version: latest
use-mamba: true
mamba-version: "*"
channels: conda-forge,e3sm/label/polaris,defaults
channel-priority: strict
auto-update-conda: true
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/docs_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,10 @@ jobs:

- if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
name: Set up Conda Environment
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: "polaris_ci"
miniforge-variant: Mambaforge
miniforge-version: latest
use-mamba: true
mamba-version: "*"
channels: conda-forge,e3sm/label/polaris,defaults
channel-priority: strict
auto-update-conda: true
Expand Down
13 changes: 6 additions & 7 deletions configure_polaris_envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
check_call,
get_conda_base,
get_logger,
install_mambaforge,
install_miniforge,
parse_args,
)

Expand Down Expand Up @@ -47,11 +47,11 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate,
if recreate or not os.path.exists(env_path):
print('Setting up a conda environment for installing polaris\n')
commands = f'{activate_base} && ' \
f'mamba create -y -n {env_name} {channels} {packages}'
f'conda create -y -n {env_name} {channels} {packages}'
else:
print('Updating conda environment for installing polaris\n')
commands = f'{activate_base} && ' \
f'mamba install -y -n {env_name} {channels} {packages}'
f'conda install -y -n {env_name} {channels} {packages}'

check_call(commands, logger=logger)

Expand All @@ -74,8 +74,7 @@ def main():
env_name = 'polaris_bootstrap'

source_activation_scripts = \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'source {conda_base}/etc/profile.d/mamba.sh'
f'source {conda_base}/etc/profile.d/conda.sh'

activate_base = f'{source_activation_scripts} && conda activate'

Expand All @@ -93,8 +92,8 @@ def main():
logger = get_logger(log_filename='deploy_tmp/logs/prebootstrap.log',
name=__name__)

# install mambaforge if needed
install_mambaforge(conda_base, activate_base, logger)
# install miniforge if needed
install_miniforge(conda_base, activate_base, logger)

local_mache = args.mache_fork is not None and args.mache_branch is not None
if local_mache:
Expand Down
19 changes: 8 additions & 11 deletions deploy/bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
get_conda_base,
get_logger,
get_spack_base,
install_mambaforge,
install_miniforge,
parse_args,
)

Expand Down Expand Up @@ -227,8 +227,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
env_path = os.path.join(conda_base, 'envs', env_name)

source_activation_scripts = \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'source {conda_base}/etc/profile.d/mamba.sh'
f'source {conda_base}/etc/profile.d/conda.sh'

activate_env = f'{source_activation_scripts} && conda activate {env_name}'

Expand All @@ -242,7 +241,7 @@ def build_conda_env(config, env_type, recreate, mpi, conda_mpi, version,
local_conda_build, logger, local_mache):

if env_type != 'dev':
install_mambaforge(conda_base, activate_base, logger)
install_miniforge(conda_base, activate_base, logger)

if conda_mpi == 'nompi':
mpi_prefix = 'nompi'
Expand Down Expand Up @@ -304,7 +303,7 @@ def build_conda_env(config, env_type, recreate, mpi, conda_mpi, version,
# install dev dependencies and polaris itself
commands = \
f'{activate_base} && ' \
f'mamba create -y -n {env_name} {channels} ' \
f'conda create -y -n {env_name} {channels} ' \
f'--file {spec_filename} {packages}'
check_call(commands, logger=logger)

Expand All @@ -319,15 +318,15 @@ def build_conda_env(config, env_type, recreate, mpi, conda_mpi, version,
version_conda = version.replace('-', '')
packages = f'{packages} "polaris={version_conda}={mpi_prefix}_*"'
commands = f'{activate_base} && ' \
f'mamba create -y -n {env_name} {channels} {packages}'
f'conda create -y -n {env_name} {channels} {packages}'
check_call(commands, logger=logger)
else:
if env_type == 'dev':
print(f'Updating {env_name}\n')
# install dev dependencies and polaris itself
commands = \
f'{activate_base} && ' \
f'mamba install -y -n {env_name} {channels} ' \
f'conda install -y -n {env_name} {channels} ' \
f'--file {spec_filename} {packages}'
check_call(commands, logger=logger)

Expand Down Expand Up @@ -921,8 +920,7 @@ def main(): # noqa: C901
conda_base = os.path.abspath(conda_base)

source_activation_scripts = \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'source {conda_base}/etc/profile.d/mamba.sh'
f'source {conda_base}/etc/profile.d/conda.sh'

activate_base = f'{source_activation_scripts} && conda activate'

Expand Down Expand Up @@ -993,7 +991,6 @@ def main(): # noqa: C901
if local_mache:
print('Install local mache\n')
commands = f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'source {conda_base}/etc/profile.d/mamba.sh && ' \
f'conda activate {conda_env_name} && ' \
'cd ../build_mache/mache && ' \
'python -m pip install .'
Expand All @@ -1008,7 +1005,7 @@ def main(): # noqa: C901
if compiler is not None:
env_vars = get_env_vars(machine, compiler, mpi)
if spack_base is not None:
spack_branch_base, spack_script, env_vars = build_spack_env(
_, spack_script, env_vars = build_spack_env(
config, args.update_spack, machine, compiler, mpi,
spack_env, spack_base, spack_template_path, env_vars,
args.tmpdir, logger)
Expand Down
3 changes: 1 addition & 2 deletions deploy/load_polaris.template
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
echo Loading conda environment
source {{ conda_base }}/etc/profile.d/conda.sh
source {{ conda_base }}/etc/profile.d/mamba.sh
mamba activate {{ polaris_env }}
conda activate {{ polaris_env }}
echo Done.
echo

Expand Down
20 changes: 10 additions & 10 deletions deploy/shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,36 +150,36 @@ def check_call(commands, env=None, logger=None):
raise subprocess.CalledProcessError(process.returncode, commands)


def install_mambaforge(conda_base, activate_base, logger):
def install_miniforge(conda_base, activate_base, logger):
if not os.path.exists(conda_base):
print('Installing Mambaforge')
print('Installing Miniforge3')
if platform.system() == 'Linux':
system = 'Linux'
elif platform.system() == 'Darwin':
system = 'MacOSX'
else:
system = 'Linux'
mambaforge = f'Mambaforge-{system}-x86_64.sh'
url = f'https://github.com/conda-forge/miniforge/releases/latest/download/{mambaforge}' # noqa: E501
miniforge = f'Miniforge3-{system}-x86_64.sh'
url = f'https://github.com/conda-forge/miniforge/releases/latest/download/{miniforge}' # noqa: E501
print(url)
req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
f = urlopen(req)
html = f.read()
with open(mambaforge, 'wb') as outfile:
with open(miniforge, 'wb') as outfile:
outfile.write(html)
f.close()

command = f'/bin/bash {mambaforge} -b -p {conda_base}'
command = f'/bin/bash {miniforge} -b -p {conda_base}'
check_call(command, logger=logger)
os.remove(mambaforge)
os.remove(miniforge)

print('Doing initial setup\n')
commands = f'{activate_base} && ' \
f'conda config --add channels conda-forge && ' \
f'conda config --set channel_priority strict && ' \
f'mamba update -y "mamba>1.3.1" "conda>=23.1.0" && ' \
f'mamba update -y --all && ' \
f'mamba init --no-user'
f'conda install -y "conda>=23.1.0" && ' \
f'conda update -y --all && ' \
f'conda init --no-user'

check_call(commands, logger=logger)

Expand Down
12 changes: 6 additions & 6 deletions docs/developers_guide/deploying_spack.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,27 +122,27 @@ cd ../simplify_local_mache/
```

You will also need a local installation of
[Mambaforge](https://github.com/conda-forge/miniforge#mambaforge).
[Miniforge3](https://github.com/conda-forge/miniforge#miniforge3).
Polaris can do this for you if you haven't already installed it. If you want
to download it manually, use the Linux x86_64 version for all our supported
machines.

:::{note}
We have found that an existing Miniconda3 installation **does not** always
work well for polaris, so please start with Mambaforge instead.
work well for polaris, so please start with Miniforge3 instead.
:::

:::{note}
You definitely need your own local Mambaforge installation -- you can’t use
You definitely need your own local Miniforge3 installation -- you can’t use
a system version or a shared one like E3SM-Unified.
:::

Define a location where Mambaforge is installed or where you want to install
Define a location where Miniforge3 is installed or where you want to install
it:

```bash
# change to your conda installation
export CONDA_BASE=${HOME}/mambaforge
export CONDA_BASE=${HOME}/miniforge3
```

Okay, we're finally ready to do a test spack build for polaris.
Expand All @@ -166,7 +166,7 @@ export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
```

The directory you point to with `--conda` either doesn't exist or contains
your existing installation of Mambaforge.
your existing installation of Miniforge3.

When you supply `--mache_fork` and `--mache_branch`, polaris will clone
a fork of the `mache` repo and check out the requested branch, then install
Expand Down
Loading

0 comments on commit 51b0efe

Please sign in to comment.