Skip to content

Commit

Permalink
copy pasted launcher from my sub scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
zaccharieramzi committed Jan 7, 2022
1 parent 38a0662 commit a1b9c41
Show file tree
Hide file tree
Showing 9 changed files with 79 additions and 0 deletions.
14 changes: 14 additions & 0 deletions jz_hydra_submitit_launcher/bin/hydra-submitit-launch
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/bin/bash
usage="$(basename "$0") [--help] python_script launch_config @hydra_args
where:
--help show this help text
python_script: the hydra python script to run
launch_config: the SLURM config to use for this run
hydra_args: arguments to pass to the hydra python script, where you can specify sweeps, but also override some launch configs"

if [ "$1" == "--help" ]; then
echo $usage
exit 0
fi

python $1 -m hydra/launcher=$2 'hydra.searchpath=[pkg://jz_hydra_submitit_launcher]' ${@:3}
5 changes: 5 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/4gpus_dev.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
defaults:
- dev

cpus_per_task: 40
gpus_per_node: 4
5 changes: 5 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/4gpus_t3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
defaults:
- t3

cpus_per_task: 40
gpus_per_node: 4
5 changes: 5 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/4gpus_t4.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
defaults:
- t4

cpus_per_task: 40
gpus_per_node: 4
17 changes: 17 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/base.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
defaults:
- submitit_slurm

timeout_min: 60
gpus_per_node: 1
tasks_per_node: 1
gres: "gpu:${hydra.launcher.gpus_per_node}"
qos: qos_gpu-dev
cpus_per_gpu: 10
gpus_per_task: ${hydra.launcher.gpus_per_node}
additional_parameters:
account: ${project_id}@gpu
distribution: "block:block"
hint: nomultithread
time: "${hours}:00:00"
setup:
- "#SBATCH -C v100-32g"
5 changes: 5 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/dev.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
defaults:
- base

additional_parameters:
time: "2:00:00"
6 changes: 6 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/t3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
defaults:
- base

qos: qos_gpu-t3
additional_parameters:
time: "20:00:00"
6 changes: 6 additions & 0 deletions jz_hydra_submitit_launcher/hydra/launcher/t4.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
defaults:
- base

qos: qos_gpu-t4
additional_parameters:
time: "100:00:00"
16 changes: 16 additions & 0 deletions jz_hydra_submitit_launcher/searchpath_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
from hydra.core.config_search_path import ConfigSearchPath
from hydra.plugins.search_path_plugin import SearchPathPlugin


class JeanZaySearchPathPlugin(SearchPathPlugin):
def manipulate_search_path(self, search_path: ConfigSearchPath) -> None:
# Appends the search path for this plugin to the end of the search path
# Note that foobar/conf is outside of the example plugin module.
# There is no requirement for it to be packaged with the plugin, it just needs
# be available in a package.
# Remember to verify the config is packaged properly (build sdist and look inside,
# and verify MANIFEST.in is correct).
search_path.append(
provider="jz-hydra-submitit-launcher", path="pkg://jz_hydra_submitit_launcher"
)

0 comments on commit a1b9c41

Please sign in to comment.