From 3e3be24b938973def6e998fa44fb9b83784ac999 Mon Sep 17 00:00:00 2001 From: Juan Medina Date: Thu, 11 Oct 2018 14:51:04 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20add=20new=20utility=20to=20regis?= =?UTF-8?q?ter=20any=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 +- Dockerfile | 1 + README.md | 4 +- register_toil/cli.py | 158 +++++++++++++++++++++++++++++++++++++++---- setup.json | 8 ++- test-container.sh | 2 +- tests/test_cli.py | 86 +++++++++++++++++------ 7 files changed, 222 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 049ca23..d19cbb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python +sudo: true + python: - 3.6 @@ -10,7 +12,7 @@ install: - pip install -U codecov script: - - bash test-container.sh + - sudo bash test-container.sh after_success: - codecov diff --git a/Dockerfile b/Dockerfile index f65f0a0..acf8a63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN \ python3-pip \ python2.7 \ python3.6 \ + uuid-runtime \ && apt-get clean \ \ # configure locale, see https://github.com/rocker-org/rocker/issues/19 diff --git a/README.md b/README.md index 9891940..e50f11c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![docker badge][automated_badge]][docker_base] [![code formatting][black_badge]][black_base] -👾 Simple utility to register versioned [toil container] pipelines in a `bin` directory. +👾 Register versioned [toil container] pipelines or other commands in singularity containers. ## Installation @@ -49,6 +49,8 @@ And the executables look like this: --volumes /ifs /ifs \ --workDir $TMP_DIR $@ +Similar usage is provided to register regular commands that will run inside a container, see `register_singularity`. + ## Contributing Contributions are welcome, and they are greatly appreciated, check our [contributing guidelines](.github/CONTRIBUTING.md)! diff --git a/register_toil/cli.py b/register_toil/cli.py index 548591f..68f8c8c 100644 --- a/register_toil/cli.py +++ b/register_toil/cli.py @@ -21,6 +21,7 @@ import subprocess import click +from slugify import slugify from register_toil import __version__ from register_toil import utils @@ -86,7 +87,7 @@ default="singularity", ) @click.version_option(version=__version__) -def main( +def register_toil( pypi_name, pypi_version, bindir, @@ -104,6 +105,11 @@ def main( bindir = Path(bindir) optexe = optdir / pypi_name binexe = bindir / f"{pypi_name}_{pypi_version}" + image_url = image_url or f"docker://leukgen/{pypi_name}:{pypi_version}" + + # check paths + assert python, "Could not determine the python path." + assert virtualenvwrapper, "Could not determine the virtualenvwrapper.sh path." # make sure dirs exist optdir.mkdir(exist_ok=True, parents=True) @@ -129,26 +135,126 @@ def main( toolpath = subprocess.check_output(["/bin/bash", "-c", install_cmd]) toolpath = toolpath.decode("utf-8").strip().split("\n")[-1] - # pull image - if not image_url: - image_url = f"docker://leukgen/{pypi_name}:{pypi_version}" - - click.echo("Pulling image...") - subprocess.check_call( - ["/bin/bash", "-c", f"umask 22 && {singularity} pull {image_url}"], cwd=optdir - ) - - # fix singularity permissions - singularity_image = next(optdir.glob("*.simg")) - singularity_image.chmod(mode=0o755) + # build command command = [ toolpath, "--singularity", - str(singularity_image), + _get_or_create_image(optdir, singularity, image_url), " ".join(f"--volumes {i} {j}" for i, j in volumes), "--workDir", tmpvar, - "$@\n", + '"$@"\n', + ] + + # link executables + click.echo("Creating and linking executable...") + optexe.write_text(f"#!/bin/bash\n{' '.join(command)}") + optexe.chmod(mode=0o755) + utils.force_symlink(optexe, binexe) + click.secho( + f"\nExecutables available at:\n" f"\n\t{str(optexe)}" f"\n\t{str(binexe)}\n", + fg="green", + ) + + +@click.command() +@click.option( + "--target", + show_default=True, + required=True, + help="name of the target script that will be created, please note that this name " + "will be slugified and the image_version will be appended (e.g. bwa_mem_pl_v1.0)", +) +@click.option( + "--command", + show_default=True, + required=True, + help="command that will be added at the end of the singularity exec instruction " + "(e.g. bwa_mem.pl)", +) +@click.option("--image_repository", required=True, help="docker hub repository name") +@click.option("--image_version", required=True, help="docker hub image version") +@click.option( + "--image_user", + default="leukgen", + help="docker hub user/organization name", + show_default=True, +) +@click.option( + "--image_url", + default=None, + help="image URL [default=docker://{image_user}/{image_repository}:{image_version}]", +) +@click.option( + "--bindir", + show_default=True, + type=click.Path(resolve_path=True, dir_okay=True), + help="path were executables will be linked to", + default=os.getenv("TOIL_REGISTER_BIN", "/work/isabl/local/bin"), +) +@click.option( + "--optdir", + show_default=True, + type=click.Path(resolve_path=True, dir_okay=True), + help="path were images will be versioned and cached", + default=os.getenv("TOIL_REGISTER_OPT", "/work/isabl/local/opt"), +) +@click.option( + "--tmpvar", + show_default=True, + help="environment variable used for --workdir", + default="$TMP_DIR", +) +@click.option( + "--volumes", + type=(click.Path(exists=True, resolve_path=True, dir_okay=True), str), + multiple=True, + default=[f"{i} {j}" for i, j in _DEFAULT_VOLUMES], + show_default=True, + help="volumes tuples to be passed to toil e.g. --volumes /juno /juno", +) +@click.option( + "--singularity", + show_default=True, + help="path to singularity", + default="singularity", +) +@click.version_option(version=__version__) +def register_singularity( # pylint: disable=R0913 + bindir, + command, + image_repository, + image_url, + image_user, + image_version, + optdir, + singularity, + target, + tmpvar, + volumes, +): + """Register versioned singularity command in a bin directory.""" + target = f"{slugify(target, separator='_')}_{image_version}" + optdir = Path(optdir) / image_repository / image_version + bindir = Path(bindir) + optexe = optdir / target + binexe = bindir / target + image_url = image_url or f"docker://{image_user}/{image_repository}:{image_version}" + + # make sure dirs exist + optdir.mkdir(exist_ok=True, parents=True) + bindir.mkdir(exist_ok=True, parents=True) + + # build command + command = [ + singularity, + "exec", + "--workdir", + f"{tmpvar}/${{USER}}_{image_repository}_{image_version}_`uuidgen`", + " ".join(f"--bind {i}:{j}" for i, j in volumes), + _get_or_create_image(optdir, singularity, image_url), + command, + '"$@"\n', ] # link executables @@ -160,3 +266,25 @@ def main( f"\nExecutables available at:\n" f"\n\t{str(optexe)}" f"\n\t{str(binexe)}\n", fg="green", ) + + +def _get_or_create_image(optdir, singularity, image_url): + # pull image + singularity_images = list(optdir.glob("*.simg")) + + assert ( + not singularity_images or len(singularity_images) == 1 + ), f"Found multiple images at {optdir}" + + if singularity_images: + click.echo(f"Image exists at: {singularity_images[0]}") + else: + subprocess.check_call( + ["/bin/bash", "-c", f"umask 22 && {singularity} pull {image_url}"], + cwd=optdir, + ) + + # fix singularity permissions + singularity_image = next(optdir.glob("*.simg")) + singularity_image.chmod(mode=0o755) + return str(singularity_image) diff --git a/setup.json b/setup.json index 4015097..b5b8a37 100644 --- a/setup.json +++ b/setup.json @@ -11,7 +11,8 @@ ], "entry_points": { "console_scripts": [ - "register_toil=register_toil.cli:main" + "register_toil=register_toil.cli:register_toil", + "register_singularity=register_toil.cli:register_singularity" ] }, "setup_requires": [ @@ -19,7 +20,8 @@ ], "install_requires": [ "Click>=6.7", - "virtualenvwrapper==4.8.2" + "virtualenvwrapper==4.8.2", + "python-slugify==1.1.2" ], "extras_require": { "test": [ @@ -37,6 +39,6 @@ "name": "register_toil", "test_suite": "tests", "long_description": "📘 learn more on `GitHub `_!", - "description": "👾 Simple utility to register versioned toil container pipelines in a bin directory.", + "description": "👾 Register versioned toil container pipelines and other commands in singularity containers.", "url": "https://github.com/leukgen/register_toil" } diff --git a/test-container.sh b/test-container.sh index 73bf0b6..3699249 100644 --- a/test-container.sh +++ b/test-container.sh @@ -25,7 +25,7 @@ find . -name '__pycache__' -exec rm -rf {} + # run tox inside the container docker run --rm $TEST_IMAGE --version -docker run --rm --entrypoint "" -v `pwd`:/test -w /test \ +docker run --privileged --rm --entrypoint "" -v `pwd`:/test -w /test \ $TEST_IMAGE bash -c "cp -r /test /register_toil && cd /register_toil && pip install tox && tox && cp .coverage /test" # move container coverage paths to local, see .coveragerc [paths] and this comment: diff --git a/tests/test_cli.py b/tests/test_cli.py index 37226b3..cc4811d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,31 +9,31 @@ from register_toil import cli -def test_main(tmpdir): - """Sample test for main command.""" +def test_register_toil(tmpdir): + """Sample test for register_toil command.""" runner = CliRunner() optdir = tmpdir.mkdir("opt") bindir = tmpdir.mkdir("bin") optexe = optdir.join("toil_disambiguate", "v0.1.2", "toil_disambiguate") binexe = bindir.join("toil_disambiguate_v0.1.2") - - params = [ - "--pypi_name", - "toil_disambiguate", - "--pypi_version", - "v0.1.2", - "--volumes", - "/tmp", - "/carlos", - "--optdir", - optdir.strpath, - "--bindir", - bindir.strpath, - "--tmpvar", - "$TMP", - ] - - result = runner.invoke(cli.main, params) + result = runner.invoke( + cli.register_toil, + [ + "--pypi_name", + "toil_disambiguate", + "--pypi_version", + "v0.1.2", + "--volumes", + "/tmp", + "/carlos", + "--optdir", + optdir.strpath, + "--bindir", + bindir.strpath, + "--tmpvar", + "$TMP", + ], + ) if result.exit_code: print(vars(result)) @@ -45,3 +45,49 @@ def test_main(tmpdir): assert "--volumes /tmp /carlos" in optexe.read() assert "--workDir $TMP" in optexe.read() + + +def test_register_command(tmpdir): + """Sample test for register_toil command.""" + runner = CliRunner() + optdir = tmpdir.mkdir("opt") + bindir = tmpdir.mkdir("bin") + optexe = optdir.join("docker-pcapcore", "v0.1.1", "bwa_mem_pl_v0.1.1") + binexe = bindir.join("bwa_mem_pl_v0.1.1") + result = runner.invoke( + cli.register_singularity, + [ + "--image_repository", + "docker-pcapcore", + "--image_version", + "v0.1.1", + "--image_user", + "leukgen", + "--volumes", + "/tmp", + "/carlos", + "--optdir", + optdir.strpath, + "--bindir", + bindir.strpath, + "--tmpvar", + "$TMP", + "--command", + "bwa_mem.pl", + "--target", + "bwa_mem.pl", + ], + ) + + if result.exit_code: + print(vars(result)) + + for i in optexe.strpath, binexe.strpath: + assert b"4.2.1" in subprocess.check_output( + args=[i, "--version"], + env={"TMP": "/tmp", "USER": "root"}, + stderr=subprocess.STDOUT, + ) + + assert "--bind /tmp:/carlos" in optexe.read() + assert "--workdir $TMP" in optexe.read()