From 645e36e6d05d62ce5690f87efd4cd8895ba2659b Mon Sep 17 00:00:00 2001 From: Jasha <8935917+Jasha10@users.noreply.github.com> Date: Tue, 18 Jan 2022 15:31:37 -0600 Subject: [PATCH 1/5] use `hydra_plugins` namespace package --- .../jz_hydra_submitit_launcher_plugin/__init__.py | 8 ++++++++ .../searchpath_config.py | 1 - jz_hydra_submitit_launcher/bin/hydra-submitit-launch | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher_plugin}/searchpath_config.py (99%) diff --git a/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py b/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py new file mode 100644 index 0000000..f090c09 --- /dev/null +++ b/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py @@ -0,0 +1,8 @@ +"""Package info""" + +__version__ = "develop" +__author__ = 'Zaccharie Ramzi' +__author_email__ = 'zaccharie.ramzi@gmail.com' +__license__ = 'MIT' +__homepage__ = 'https://github.com/zaccharieramzi/jz-hydra-submitit-launcher' +__docs__ = 'Jean Zay tailored Hydra submitit launcher.' diff --git a/jz_hydra_submitit_launcher/searchpath_config.py b/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py similarity index 99% rename from jz_hydra_submitit_launcher/searchpath_config.py rename to hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py index 740568e..cea3102 100644 --- a/jz_hydra_submitit_launcher/searchpath_config.py +++ b/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py @@ -2,7 +2,6 @@ from hydra.core.config_search_path import ConfigSearchPath from hydra.plugins.search_path_plugin import SearchPathPlugin - class JeanZaySearchPathPlugin(SearchPathPlugin): def manipulate_search_path(self, search_path: ConfigSearchPath) -> None: # Appends the search path for this plugin to the end of the search path diff --git a/jz_hydra_submitit_launcher/bin/hydra-submitit-launch b/jz_hydra_submitit_launcher/bin/hydra-submitit-launch index 5cf08e1..46acc48 100644 --- a/jz_hydra_submitit_launcher/bin/hydra-submitit-launch +++ b/jz_hydra_submitit_launcher/bin/hydra-submitit-launch @@ -11,4 +11,4 @@ if [ "$1" == "--help" ]; then exit 0 fi -python $1 -m hydra/launcher=$2 'hydra.searchpath=[pkg://jz_hydra_submitit_launcher]' "${@:3}" \ No newline at end of file +python $1 -m hydra/launcher=$2 "${@:3}" \ No newline at end of file From adbd7ab23986b779ab1bfc352fe10cd40b29e44f Mon Sep 17 00:00:00 2001 From: Jasha <8935917+Jasha10@users.noreply.github.com> Date: Tue, 18 Jan 2022 15:35:25 -0600 Subject: [PATCH 2/5] restore whitespace --- .../jz_hydra_submitit_launcher_plugin/searchpath_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py b/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py index cea3102..740568e 100644 --- a/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py +++ b/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py @@ -2,6 +2,7 @@ from hydra.core.config_search_path import ConfigSearchPath from hydra.plugins.search_path_plugin import SearchPathPlugin + class JeanZaySearchPathPlugin(SearchPathPlugin): def manipulate_search_path(self, search_path: ConfigSearchPath) -> None: # Appends the search path for this plugin to the end of the search path From 00717f0367e61c52bba76f915811bfde8af22b75 Mon Sep 17 00:00:00 2001 From: Jasha <8935917+Jasha10@users.noreply.github.com> Date: Tue, 18 Jan 2022 15:37:49 -0600 Subject: [PATCH 3/5] remove data from copied __init__.py file --- .../jz_hydra_submitit_launcher_plugin/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py b/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py index f090c09..e69de29 100644 --- a/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py +++ b/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py @@ -1,8 +0,0 @@ -"""Package info""" - -__version__ = "develop" -__author__ = 'Zaccharie Ramzi' -__author_email__ = 'zaccharie.ramzi@gmail.com' -__license__ = 'MIT' -__homepage__ = 'https://github.com/zaccharieramzi/jz-hydra-submitit-launcher' -__docs__ = 'Jean Zay tailored Hydra submitit launcher.' From 2ec13ab25f8d2be13778fd27dc88db3e4aa28f98 Mon Sep 17 00:00:00 2001 From: Jasha <8935917+Jasha10@users.noreply.github.com> Date: Wed, 19 Jan 2022 16:09:25 -0600 Subject: [PATCH 4/5] move everything into hydra_plugins --- MANIFEST.in | 2 +- .../jz_hydra_submitit_launcher}/__init__.py | 0 .../jz_hydra_submitit_launcher}/bin/hydra-submitit-launch | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/4gpus_dev.yaml | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/4gpus_t3.yaml | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/4gpus_t4.yaml | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/base.yaml | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/dev.yaml | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/t3.yaml | 0 .../jz_hydra_submitit_launcher}/hydra/launcher/t4.yaml | 0 hydra_plugins/jz_hydra_submitit_launcher/resolvers.py | 3 +++ .../searchpath_config.py | 2 +- hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py | 0 setup.py | 4 ++-- 14 files changed, 7 insertions(+), 4 deletions(-) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/__init__.py (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/bin/hydra-submitit-launch (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/4gpus_dev.yaml (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/4gpus_t3.yaml (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/4gpus_t4.yaml (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/base.yaml (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/dev.yaml (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/t3.yaml (100%) rename {jz_hydra_submitit_launcher => hydra_plugins/jz_hydra_submitit_launcher}/hydra/launcher/t4.yaml (100%) create mode 100644 hydra_plugins/jz_hydra_submitit_launcher/resolvers.py rename hydra_plugins/{jz_hydra_submitit_launcher_plugin => jz_hydra_submitit_launcher}/searchpath_config.py (88%) delete mode 100644 hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 48d1458..f155819 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ global-exclude *.pyc global-exclude __pycache__ -recursive-include jz_hydra_submitit_launcher/* *.yaml \ No newline at end of file +recursive-include hydra_plugins/* *.yaml \ No newline at end of file diff --git a/jz_hydra_submitit_launcher/__init__.py b/hydra_plugins/jz_hydra_submitit_launcher/__init__.py similarity index 100% rename from jz_hydra_submitit_launcher/__init__.py rename to hydra_plugins/jz_hydra_submitit_launcher/__init__.py diff --git a/jz_hydra_submitit_launcher/bin/hydra-submitit-launch b/hydra_plugins/jz_hydra_submitit_launcher/bin/hydra-submitit-launch similarity index 100% rename from jz_hydra_submitit_launcher/bin/hydra-submitit-launch rename to hydra_plugins/jz_hydra_submitit_launcher/bin/hydra-submitit-launch diff --git a/jz_hydra_submitit_launcher/hydra/launcher/4gpus_dev.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/4gpus_dev.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/4gpus_dev.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/4gpus_dev.yaml diff --git a/jz_hydra_submitit_launcher/hydra/launcher/4gpus_t3.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/4gpus_t3.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/4gpus_t3.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/4gpus_t3.yaml diff --git a/jz_hydra_submitit_launcher/hydra/launcher/4gpus_t4.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/4gpus_t4.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/4gpus_t4.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/4gpus_t4.yaml diff --git a/jz_hydra_submitit_launcher/hydra/launcher/base.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/base.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/base.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/base.yaml diff --git a/jz_hydra_submitit_launcher/hydra/launcher/dev.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/dev.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/dev.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/dev.yaml diff --git a/jz_hydra_submitit_launcher/hydra/launcher/t3.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/t3.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/t3.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/t3.yaml diff --git a/jz_hydra_submitit_launcher/hydra/launcher/t4.yaml b/hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/t4.yaml similarity index 100% rename from jz_hydra_submitit_launcher/hydra/launcher/t4.yaml rename to hydra_plugins/jz_hydra_submitit_launcher/hydra/launcher/t4.yaml diff --git a/hydra_plugins/jz_hydra_submitit_launcher/resolvers.py b/hydra_plugins/jz_hydra_submitit_launcher/resolvers.py new file mode 100644 index 0000000..1cb898c --- /dev/null +++ b/hydra_plugins/jz_hydra_submitit_launcher/resolvers.py @@ -0,0 +1,3 @@ +from omegaconf import OmegaConf + +OmegaConf.register_new_resolver("multiply10", lambda x: x * 10) diff --git a/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py b/hydra_plugins/jz_hydra_submitit_launcher/searchpath_config.py similarity index 88% rename from hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py rename to hydra_plugins/jz_hydra_submitit_launcher/searchpath_config.py index 740568e..664654e 100644 --- a/hydra_plugins/jz_hydra_submitit_launcher_plugin/searchpath_config.py +++ b/hydra_plugins/jz_hydra_submitit_launcher/searchpath_config.py @@ -12,5 +12,5 @@ def manipulate_search_path(self, search_path: ConfigSearchPath) -> None: # Remember to verify the config is packaged properly (build sdist and look inside, # and verify MANIFEST.in is correct). search_path.append( - provider="jz-hydra-submitit-launcher", path="pkg://jz_hydra_submitit_launcher" + provider="jz-hydra-submitit-launcher", path="pkg://hydra_plugins.jz_hydra_submitit_launcher" ) \ No newline at end of file diff --git a/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py b/hydra_plugins/jz_hydra_submitit_launcher_plugin/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index d0e779d..87edf5b 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open('requirements.txt') as open_file: install_requires = open_file.read() -import jz_hydra_submitit_launcher +from hydra_plugins import jz_hydra_submitit_launcher setuptools.setup( name="jz-hydra-submitit-launcher", @@ -31,5 +31,5 @@ python_requires='>=3.6', include_package_data=True, keywords=['hydra', 'submitit', 'jean-zay'], - scripts=['jz_hydra_submitit_launcher/bin/hydra-submitit-launch'], + scripts=['hydra_plugins/jz_hydra_submitit_launcher/bin/hydra-submitit-launch'], ) From dd7a1bc25f32e1ac73cf6287493b8770e70a0219 Mon Sep 17 00:00:00 2001 From: Jasha <8935917+Jasha10@users.noreply.github.com> Date: Wed, 19 Jan 2022 16:17:42 -0600 Subject: [PATCH 5/5] make MANIFEST more specific --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index f155819..e9dc594 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ global-exclude *.pyc global-exclude __pycache__ -recursive-include hydra_plugins/* *.yaml \ No newline at end of file +recursive-include hydra_plugins/jz_hydra_submitit_launcher/* *.yaml \ No newline at end of file