🚀 add new utility to register any command

.travis.yml

@@ -1,5 +1,7 @@
 language: python
 
+sudo: true
+
 python:
  - 3.6
 
@@ -10,7 +12,7 @@ install:
  - pip install -U codecov
 
 script:
- - bash test-container.sh
+ - sudo bash test-container.sh
 
 after_success:
  - codecov

Dockerfile

@@ -15,6 +15,7 @@ RUN \
  python3-pip \
  python2.7 \
  python3.6 \
+ uuid-runtime \
  && apt-get clean \
  \
  # configure locale, see https://github.com/rocker-org/rocker/issues/19

README.md

@@ -7,7 +7,7 @@
 [![docker badge][automated_badge]][docker_base]
 [![code formatting][black_badge]][black_base]
 
-👾 Simple utility to register versioned [toil container] pipelines in a `bin` directory.
+👾 Register versioned [toil container] pipelines or other commands in singularity containers.
 
 ## Installation
 
@@ -49,6 +49,8 @@ And the executables look like this:
   --volumes /ifs /ifs \
   --workDir $TMP_DIR $@
 
+Similar usage is provided to register regular commands that will run inside a container, see `register_singularity`.
+
 ## Contributing
 
 Contributions are welcome, and they are greatly appreciated, check our [contributing guidelines](.github/CONTRIBUTING.md)!

register_toil/cli.py

@@ -21,6 +21,7 @@
 import subprocess
 
 import click
+from slugify import slugify
 
 from register_toil import __version__
 from register_toil import utils
@@ -86,7 +87,7 @@
  default="singularity",
 )
 @click.version_option(version=__version__)
-def main(
+def register_toil(
  pypi_name,
  pypi_version,
  bindir,
@@ -104,6 +105,11 @@ def main(
  bindir = Path(bindir)
  optexe = optdir / pypi_name
  binexe = bindir / f"{pypi_name}_{pypi_version}"
+ image_url = image_url or f"docker://leukgen/{pypi_name}:{pypi_version}"
+
+ # check paths
+ assert python, "Could not determine the python path."
+ assert virtualenvwrapper, "Could not determine the virtualenvwrapper.sh path."
 
  # make sure dirs exist
  optdir.mkdir(exist_ok=True, parents=True)
@@ -129,26 +135,126 @@ def main(
  toolpath = subprocess.check_output(["/bin/bash", "-c", install_cmd])
  toolpath = toolpath.decode("utf-8").strip().split("\n")[-1]
 
- # pull image
- if not image_url:
- image_url = f"docker://leukgen/{pypi_name}:{pypi_version}"
-
- click.echo("Pulling image...")
- subprocess.check_call(
- ["/bin/bash", "-c", f"umask 22 && {singularity} pull {image_url}"], cwd=optdir
- )
-
- # fix singularity permissions
- singularity_image = next(optdir.glob("*.simg"))
- singularity_image.chmod(mode=0o755)
+ # build command
  command = [
  toolpath,
  "--singularity",
- str(singularity_image),
+ _get_or_create_image(optdir, singularity, image_url),
  " ".join(f"--volumes {i} {j}" for i, j in volumes),
  "--workDir",
  tmpvar,
- "$@\n",
+ '"$@"\n',
+ ]
+
+ # link executables
+ click.echo("Creating and linking executable...")
+ optexe.write_text(f"#!/bin/bash\n{' '.join(command)}")
+ optexe.chmod(mode=0o755)
+ utils.force_symlink(optexe, binexe)
+ click.secho(
+ f"\nExecutables available at:\n" f"\n\t{str(optexe)}" f"\n\t{str(binexe)}\n",
+ fg="green",
+ )
+
+
+@click.command()
+@click.option(
+ "--target",
+ show_default=True,
+ required=True,
+ help="name of the target script that will be created, please note that this name "
+ "will be slugified and the image_version will be appended (e.g. bwa_mem_pl_v1.0)",
+)
+@click.option(
+ "--command",
+ show_default=True,
+ required=True,
+ help="command that will be added at the end of the singularity exec instruction "
+ "(e.g. bwa_mem.pl)",
+)
+@click.option("--image_repository", required=True, help="docker hub repository name")
+@click.option("--image_version", required=True, help="docker hub image version")
+@click.option(
+ "--image_user",
+ default="leukgen",
+ help="docker hub user/organization name",
+ show_default=True,
+)
+@click.option(
+ "--image_url",
+ default=None,
+ help="image URL [default=docker://{image_user}/{image_repository}:{image_version}]",
+)
+@click.option(
+ "--bindir",
+ show_default=True,
+ type=click.Path(resolve_path=True, dir_okay=True),
+ help="path were executables will be linked to",
+ default=os.getenv("TOIL_REGISTER_BIN", "/work/isabl/local/bin"),
+)
+@click.option(
+ "--optdir",
+ show_default=True,
+ type=click.Path(resolve_path=True, dir_okay=True),
+ help="path were images will be versioned and cached",
+ default=os.getenv("TOIL_REGISTER_OPT", "/work/isabl/local/opt"),
+)
+@click.option(
+ "--tmpvar",
+ show_default=True,
+ help="environment variable used for --workdir",
+ default="$TMP_DIR",
+)
+@click.option(
+ "--volumes",
+ type=(click.Path(exists=True, resolve_path=True, dir_okay=True), str),
+ multiple=True,
+ default=[f"{i} {j}" for i, j in _DEFAULT_VOLUMES],
+ show_default=True,
+ help="volumes tuples to be passed to toil e.g. --volumes /juno /juno",
+)
+@click.option(
+ "--singularity",
+ show_default=True,
+ help="path to singularity",
+ default="singularity",
+)
+@click.version_option(version=__version__)
+def register_singularity( # pylint: disable=R0913
+ bindir,
+ command,
+ image_repository,
+ image_url,
+ image_user,
+ image_version,
+ optdir,
+ singularity,
+ target,
+ tmpvar,
+ volumes,
+):
+ """Register versioned singularity command in a bin directory."""
+ target = f"{slugify(target, separator='_')}_{image_version}"
+ optdir = Path(optdir) / image_repository / image_version
+ bindir = Path(bindir)
+ optexe = optdir / target
+ binexe = bindir / target
+ image_url = image_url or f"docker://{image_user}/{image_repository}:{image_version}"
+
+ # make sure dirs exist
+ optdir.mkdir(exist_ok=True, parents=True)
+ bindir.mkdir(exist_ok=True, parents=True)
+
+ # build command
+ command = [
+ singularity,
+ "exec",
+ "--workdir",
+ f"{tmpvar}/${{USER}}_{image_repository}_{image_version}_`uuidgen`",
+ " ".join(f"--bind {i}:{j}" for i, j in volumes),
+ _get_or_create_image(optdir, singularity, image_url),
+ command,
+ '"$@"\n',
  ]
 
  # link executables
@@ -160,3 +266,25 @@ def main(
  f"\nExecutables available at:\n" f"\n\t{str(optexe)}" f"\n\t{str(binexe)}\n",
  fg="green",
  )
+
+
+def _get_or_create_image(optdir, singularity, image_url):
+ # pull image
+ singularity_images = list(optdir.glob("*.simg"))
+
+ assert (
+ not singularity_images or len(singularity_images) == 1
+ ), f"Found multiple images at {optdir}"
+
+ if singularity_images:
+ click.echo(f"Image exists at: {singularity_images[0]}")
+ else:
+ subprocess.check_call(
+ ["/bin/bash", "-c", f"umask 22 && {singularity} pull {image_url}"],
+ cwd=optdir,
+ )
+
+ # fix singularity permissions
+ singularity_image = next(optdir.glob("*.simg"))
+ singularity_image.chmod(mode=0o755)
+ return str(singularity_image)

setup.json

@@ -11,15 +11,17 @@
  ],
  "entry_points": {
  "console_scripts": [
- "register_toil=register_toil.cli:main"
+ "register_toil=register_toil.cli:register_toil",
+ "register_singularity=register_toil.cli:register_singularity"
  ]
  },
  "setup_requires": [
  "pytest-runner==2.11.1"
  ],
  "install_requires": [
  "Click>=6.7",
- "virtualenvwrapper==4.8.2"
+ "virtualenvwrapper==4.8.2",
+ "python-slugify==1.1.2"
  ],
  "extras_require": {
  "test": [
@@ -37,6 +39,6 @@
  "name": "register_toil",
  "test_suite": "tests",
  "long_description": "📘 learn more on `GitHub <https://github.com/leukgen/register_toil>`_!",
- "description": "👾 Simple utility to register versioned toil container pipelines in a bin directory.",
+ "description": "👾 Register versioned toil container pipelines and other commands in singularity containers.",
  "url": "https://github.com/leukgen/register_toil"
 }

test-container.sh

@@ -25,7 +25,7 @@ find . -name '__pycache__' -exec rm -rf {} +
 
 # run tox inside the container
 docker run --rm $TEST_IMAGE --version
-docker run --rm --entrypoint "" -v `pwd`:/test -w /test \
+docker run --privileged --rm --entrypoint "" -v `pwd`:/test -w /test \
  $TEST_IMAGE bash -c "cp -r /test /register_toil && cd /register_toil && pip install tox && tox && cp .coverage /test"
 
 # move container coverage paths to local, see .coveragerc [paths] and this comment:

tests/test_cli.py

@@ -9,31 +9,31 @@
 from register_toil import cli
 
 
-def test_main(tmpdir):
- """Sample test for main command."""
+def test_register_toil(tmpdir):
+ """Sample test for register_toil command."""
  runner = CliRunner()
  optdir = tmpdir.mkdir("opt")
  bindir = tmpdir.mkdir("bin")
  optexe = optdir.join("toil_disambiguate", "v0.1.2", "toil_disambiguate")
  binexe = bindir.join("toil_disambiguate_v0.1.2")
-
- params = [
- "--pypi_name",
- "toil_disambiguate",
- "--pypi_version",
- "v0.1.2",
- "--volumes",
- "/tmp",
- "/carlos",
- "--optdir",
- optdir.strpath,
- "--bindir",
- bindir.strpath,
- "--tmpvar",
- "$TMP",
- ]
-
- result = runner.invoke(cli.main, params)
+ result = runner.invoke(
+  cli.register_toil,
+ [
+  "--pypi_name",
+  "toil_disambiguate",
+  "--pypi_version",
+  "v0.1.2",
+  "--volumes",
+  "/tmp",
+  "/carlos",
+  "--optdir",
+  optdir.strpath,
+  "--bindir",
+  bindir.strpath,
+  "--tmpvar",
+  "$TMP",
+ ],
+ )
 
  if result.exit_code:
  print(vars(result))
@@ -45,3 +45,49 @@ def test_main(tmpdir):
 
  assert "--volumes /tmp /carlos" in optexe.read()
  assert "--workDir $TMP" in optexe.read()
+
+
+def test_register_command(tmpdir):
+ """Sample test for register_toil command."""
+ runner = CliRunner()
+ optdir = tmpdir.mkdir("opt")
+ bindir = tmpdir.mkdir("bin")
+ optexe = optdir.join("docker-pcapcore", "v0.1.1", "bwa_mem_pl_v0.1.1")
+ binexe = bindir.join("bwa_mem_pl_v0.1.1")
+ result = runner.invoke(
+ cli.register_singularity,
+ [
+ "--image_repository",
+ "docker-pcapcore",
+ "--image_version",
+ "v0.1.1",
+ "--image_user",
+ "leukgen",
+ "--volumes",
+ "/tmp",
+ "/carlos",
+ "--optdir",
+ optdir.strpath,
+ "--bindir",
+ bindir.strpath,
+ "--tmpvar",
+ "$TMP",
+ "--command",
+ "bwa_mem.pl",
+ "--target",
+ "bwa_mem.pl",
+ ],
+ )
+
+ if result.exit_code:
+ print(vars(result))
+
+ for i in optexe.strpath, binexe.strpath:
+ assert b"4.2.1" in subprocess.check_output(
+ args=[i, "--version"],
+ env={"TMP": "/tmp", "USER": "root"},
+ stderr=subprocess.STDOUT,
+ )
+
+ assert "--bind /tmp:/carlos" in optexe.read()
+ assert "--workdir $TMP" in optexe.read()