From 77260d056554c7c3811b2f2280a68da4343475f7 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 16:28:37 -0400 Subject: [PATCH 01/27] =?UTF-8?q?:truck:=20Update=20badges:=20`master`=20?= =?UTF-8?q?=E2=86=92=20`trunk`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 73be5d7..1b46167 100644 --- a/README.rst +++ b/README.rst @@ -70,9 +70,9 @@ Usage .. |github-version| image:: https://img.shields.io/github/tag/shnizzedy/cpac-python-package.svg :target: https://github.com/shnizzedy/cpac-python-package/releases :alt: GitHub version -.. |build-status| image:: https://travis-ci.org/shnizzedy/cpac-python-package.svg?branch=master +.. |build-status| image:: https://travis-ci.org/shnizzedy/cpac-python-package.svg?branch=trunk :target: https://travis-ci.org/shnizzedy/cpac-python-package :alt: Travis CI build status -.. |coverage| image:: https://coveralls.io/repos/github/shnizzedy/cpac-python-package/badge.svg?branch=master - :target: https://coveralls.io/github/shnizzedy/cpac-python-package?branch=master +.. |coverage| image:: https://coveralls.io/repos/github/shnizzedy/cpac-python-package/badge.svg?branch=trunk + :target: https://coveralls.io/github/shnizzedy/cpac-python-package?branch=trunk :alt: coverage badge From bea659ea540a32ccdc7dc7fd697aadea50a4a2bc Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 15:40:34 -0400 Subject: [PATCH 02/27] :whale: Fix a bug where Docker would not print with the command `cpac crash` Ref #34 --- src/cpac/backends/docker.py | 60 +++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/cpac/backends/docker.py b/src/cpac/backends/docker.py index 2f337ba..133680c 100644 --- a/src/cpac/backends/docker.py +++ b/src/cpac/backends/docker.py @@ -91,32 +91,48 @@ def _execute(self, command, run_type='run', **kwargs): ) for k in layer if k in {'id', 'status', 'progress'}] self._load_logging() - print(command) - self.container = self.client.containers.run( - self.image, - command=command, - detach=True, - stderr=True, - stdout=True, - remove=True, - user=':'.join([ - str(self.bindings['uid']), - str(self.bindings['gid']) - ]), - volumes=self._volumes_to_docker_mounts(), - working_dir=kwargs.get('working_dir', '/tmp'), - **self.docker_kwargs - ) - if run_type == 'exec': - return(self.container.attach( - logs=True, stdout=True, stderr=True, stream=True - )) - else: + + if run_type == 'run': + self.container = self.client.containers.run( + self.image, + command=command, + detach=True, + stderr=True, + stdout=True, + remove=True, + user=':'.join([ + str(self.bindings['uid']), + str(self.bindings['gid']) + ]), + volumes=self._volumes_to_docker_mounts(), + working_dir=kwargs.get('working_dir', '/tmp'), + **self.docker_kwargs + ) self._run = DockerRun(self.container) + elif run_type == 'exec': + self.container = self.client.containers.create( + self.image, + auto_remove=True, + entrypoint='/bin/bash', + stdin_open=True, + user=':'.join([ + str(self.bindings['uid']), + str(self.bindings['gid']) + ]), + volumes=self._volumes_to_docker_mounts(), + working_dir=kwargs.get('working_dir', '/tmp'), + **self.docker_kwargs + ) + self.container.start() + return(self.container.exec_run( + cmd=command, + stdout=True, + stderr=True, + stream=True + )[1]) class DockerRun(object): - def __init__(self, container): self.container = container [print(l.decode('utf-8'), end='') for l in self.container.attach( From 1929a85e80c8020d5eeda025fb3de0cc56ceee0c Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 16:22:05 -0400 Subject: [PATCH 03/27] :books: Update CHANGELOG for resolving #34 --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 765cfd6..a2aa180 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ Changelog `Version 0.2.5 `_ ======================================================================================= * 📢🐳 Provide a clearer error message if package cannot connect to Docker. +* 🐳 Fix a bug introduced in `v0.2.4 ` where some crashfiles would print for ``cpac --platform singularity crash`` but not for ``cpac --platform docker crash`` `Version 0.2.4 `_ ======================================================================================= From 299b0fcd266cc6574d735fd80e12e24a0a05c37e Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 16:22:27 -0400 Subject: [PATCH 04/27] :pencil: Improve CI test for `cpac crash` --- tests/test_cpac_crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpac_crash.py b/tests/test_cpac_crash.py index 27bad58..47be644 100644 --- a/tests/test_cpac_crash.py +++ b/tests/test_cpac_crash.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize('args', PLATFORM_ARGS) -def test_utils_new_settings_template(args, capsys): +def test_cpac_crash(args, capsys): crashfile = os.path.join( os.path.dirname(__file__), 'test_data', 'test_pickle.pklz' ) From f64a04c947ea9949d1ea7d44f198582e89691884 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 13 Jul 2020 16:51:59 -0400 Subject: [PATCH 05/27] :alembic: Run coverage with `--append` option --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0df3206..159b072 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,8 @@ before_script: - git config --global user.name "Travis" - singularity pull shub://FCP-INDI/C-PAC || singularity pull docker://fcpindi/c-pac:latest script: - - coverage run -m pytest + - coverage run -m pytest --append + - coverage combine - coverage report -m after_success: - if [[ "$COVERAGE" == "true" ]]; then coveralls || echo "failed"; fi From 10a3104e21fc3b60c45ec91bddaf276959186b84 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 13 Jul 2020 19:08:33 -0400 Subject: [PATCH 06/27] :name_badge: Set metadataname to 'cpac' to match CLI name --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 10947f1..961895d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ # http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files [metadata] -name = cpac_py +name = cpac description = C-PAC Python Package author = C-PAC developers author-email = cpac@cnl.childmind.org @@ -41,6 +41,7 @@ install_requires = nipype pandas >= 0.23.4 spython >= 0.0.81 + pyyaml tabulate >= 0.8.6 tornado websocket-client From 332eddc4f69550e483f670653da33e25596b2dde Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 14 Jul 2020 10:05:42 -0400 Subject: [PATCH 07/27] fixup! :alembic: Run coverage with `--append` option --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 159b072..3a0435d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ before_script: - git config --global user.name "Travis" - singularity pull shub://FCP-INDI/C-PAC || singularity pull docker://fcpindi/c-pac:latest script: - - coverage run -m pytest --append + - coverage --append run -m pytest - coverage combine - coverage report -m after_success: From e92c336e47ea182dbe8ce46a51999ae1c76adc65 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 14 Jul 2020 10:25:50 -0400 Subject: [PATCH 08/27] fixup! fixup! :alembic: Run coverage with `--append` option --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a0435d..05aa268 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ before_script: - git config --global user.name "Travis" - singularity pull shub://FCP-INDI/C-PAC || singularity pull docker://fcpindi/c-pac:latest script: - - coverage --append run -m pytest + - coverage run --append -m pytest - coverage combine - coverage report -m after_success: From c2e5ad4e57a6cc1135eceace43ad9fceeb832765 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 14 Jul 2020 10:47:29 -0400 Subject: [PATCH 09/27] fixup! fixup! fixup! :alembic: Run coverage with `--append` option --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 05aa268..22bc509 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,6 @@ before_script: - singularity pull shub://FCP-INDI/C-PAC || singularity pull docker://fcpindi/c-pac:latest script: - coverage run --append -m pytest - - coverage combine - coverage report -m after_success: - if [[ "$COVERAGE" == "true" ]]; then coveralls || echo "failed"; fi From 1060923724eac9744559f22c939950eb86f4018c Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 15:40:34 -0400 Subject: [PATCH 10/27] :whale: Fix a bug where Docker would not print with the command `cpac crash` Ref #34 --- src/cpac/backends/docker.py | 60 +++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/cpac/backends/docker.py b/src/cpac/backends/docker.py index 2f337ba..133680c 100644 --- a/src/cpac/backends/docker.py +++ b/src/cpac/backends/docker.py @@ -91,32 +91,48 @@ def _execute(self, command, run_type='run', **kwargs): ) for k in layer if k in {'id', 'status', 'progress'}] self._load_logging() - print(command) - self.container = self.client.containers.run( - self.image, - command=command, - detach=True, - stderr=True, - stdout=True, - remove=True, - user=':'.join([ - str(self.bindings['uid']), - str(self.bindings['gid']) - ]), - volumes=self._volumes_to_docker_mounts(), - working_dir=kwargs.get('working_dir', '/tmp'), - **self.docker_kwargs - ) - if run_type == 'exec': - return(self.container.attach( - logs=True, stdout=True, stderr=True, stream=True - )) - else: + + if run_type == 'run': + self.container = self.client.containers.run( + self.image, + command=command, + detach=True, + stderr=True, + stdout=True, + remove=True, + user=':'.join([ + str(self.bindings['uid']), + str(self.bindings['gid']) + ]), + volumes=self._volumes_to_docker_mounts(), + working_dir=kwargs.get('working_dir', '/tmp'), + **self.docker_kwargs + ) self._run = DockerRun(self.container) + elif run_type == 'exec': + self.container = self.client.containers.create( + self.image, + auto_remove=True, + entrypoint='/bin/bash', + stdin_open=True, + user=':'.join([ + str(self.bindings['uid']), + str(self.bindings['gid']) + ]), + volumes=self._volumes_to_docker_mounts(), + working_dir=kwargs.get('working_dir', '/tmp'), + **self.docker_kwargs + ) + self.container.start() + return(self.container.exec_run( + cmd=command, + stdout=True, + stderr=True, + stream=True + )[1]) class DockerRun(object): - def __init__(self, container): self.container = container [print(l.decode('utf-8'), end='') for l in self.container.attach( From a196134576fae7412834a9f99413bf6b60f8ec90 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 16:22:05 -0400 Subject: [PATCH 11/27] :books: Update CHANGELOG for resolving #34 --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 765cfd6..a2aa180 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ Changelog `Version 0.2.5 `_ ======================================================================================= * 📢🐳 Provide a clearer error message if package cannot connect to Docker. +* 🐳 Fix a bug introduced in `v0.2.4 ` where some crashfiles would print for ``cpac --platform singularity crash`` but not for ``cpac --platform docker crash`` `Version 0.2.4 `_ ======================================================================================= From 0e178225477d014636b04de4a80f6272271ee32c Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 10 Jul 2020 16:22:27 -0400 Subject: [PATCH 12/27] :pencil: Improve CI test for `cpac crash` --- tests/test_cpac_crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpac_crash.py b/tests/test_cpac_crash.py index 27bad58..47be644 100644 --- a/tests/test_cpac_crash.py +++ b/tests/test_cpac_crash.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize('args', PLATFORM_ARGS) -def test_utils_new_settings_template(args, capsys): +def test_cpac_crash(args, capsys): crashfile = os.path.join( os.path.dirname(__file__), 'test_data', 'test_pickle.pklz' ) From 599b90e6e7361fb33d0e19941121175f4e4bc047 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 17 Jul 2020 16:58:46 -0400 Subject: [PATCH 13/27] :hammer: Separate dev requirements from install requirements --- .travis.yml | 1 + requirements-dev.txt | 5 +++++ requirements.txt | 7 +------ 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 requirements-dev.txt diff --git a/.travis.yml b/.travis.yml index 0df3206..cd26742 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ before_install: install: - source tests/travis_install.sh - pip install -r requirements.txt + - pip install -r requirements-dev.txt - pip install twine - pip install . before_script: diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..6196be4 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +coveralls +pytest +pytest-remotedata >= 0.3.2 +pytest-runner +sphinx \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1a86f3a..b4807e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,10 @@ -coveralls docker >= 4.2.1 docker-pycreds nipype pandas >= 0.23.4 -pytest -pytest-remotedata>=0.3.2 -pytest-runner pyyaml setuptools -sphinx spython >= 0.0.81 tabulate >= 0.8.6 tornado -websocket-client +websocket-client \ No newline at end of file From 373c8c91546be576aba9ee0ce89e5d86a6957188 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 17 Jul 2020 16:59:07 -0400 Subject: [PATCH 14/27] :microscope: Test installation requirements --- tests/test_cpac_installation.py | 90 +++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/test_cpac_installation.py diff --git a/tests/test_cpac_installation.py b/tests/test_cpac_installation.py new file mode 100644 index 0000000..2606472 --- /dev/null +++ b/tests/test_cpac_installation.py @@ -0,0 +1,90 @@ +import setuptools + +from pip._internal.utils.misc import get_installed_distributions +from setuptools.config import read_configuration + + +def test_requirements(): + requirements = { + 'setup.cfg': requirements_list(read_configuration( + 'setup.cfg' + )['options']['install_requires'] + ) + } + with open('requirements.txt', 'r') as req: + requirements['requirements.txt']=requirements_list( + req.readlines() + ) + for req in requirements['requirements.txt']: + if req.package.lower() != 'setuptools': + assert package_in_list( + req, requirements['setup.cfg'] + ), ( + f'package {req} is in requirements.txt ' + 'but not in setup.cfg' + ) + assert package_in_list( + req, requirements_list(get_installed_distributions() + )), ( + f'package {req} is in requirements.txt ' + 'but not installed' + ) + + +class Requirement(): + def __init__(self, requirement): + package = str(requirement).rstrip().split(' ') + self.package = package[0] + self.version = { + "==": package[1] + } if len(package) == 2 else { + package[i]: package[i+1] for i in range(len(package)) if i%2 + } + + def __repr__(self): + return ' '.join([ + f'{self.package}', + ', '.join([ + f'{key} {self.version[key]}' for key in self.version + ]) + ]).strip() + + +def package_in_list(package, version_list): + """ + Helper function to check if a case-insensitive named package + is included in a list of Requirements + + Parameters + ---------- + package: str + + version_list: list + + Returns + ------- + bool + """ + return( + package.package.lower() in [ + p.package.lower() for p in version_list + ] + ) + + +def requirements_list(requirements): + """ + Helper function to split coerce a list of requirements into + a list of Requirements + + Parameters + ---------- + requirements: list + list of requirment strings + + Returns + ------- + list + list of Requirements + """ + return([Requirement(r) for r in requirements]) \ No newline at end of file From e1b5856ecff178b72119d28ada55dcdd719af90d Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 17 Jul 2020 17:05:19 -0400 Subject: [PATCH 15/27] :microscope: Test version --- tests/test_cpac_installation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cpac_installation.py b/tests/test_cpac_installation.py index 2606472..454f23e 100644 --- a/tests/test_cpac_installation.py +++ b/tests/test_cpac_installation.py @@ -29,6 +29,11 @@ def test_requirements(): f'package {req} is in requirements.txt ' 'but not installed' ) + + +def test_version(): + from cpac import __version__ + assert __version__ != 'undefined', f'version is {__version__}' class Requirement(): From 482f3786f24528d6b4bb90613d30cafbaa65e165 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 17 Jul 2020 17:07:50 -0400 Subject: [PATCH 16/27] =?UTF-8?q?:books:=20Update=20CHANGELOG:=20?= =?UTF-8?q?=F0=9F=9A=91=20Fix=20some=20installation=20issues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a2aa180..f878910 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ Changelog ======================================================================================= * 📢🐳 Provide a clearer error message if package cannot connect to Docker. * 🐳 Fix a bug introduced in `v0.2.4 ` where some crashfiles would print for ``cpac --platform singularity crash`` but not for ``cpac --platform docker crash`` +* 🚑 Fix some installation issues: + * All required packages are now installed with ``pip install cpac-py`` + * Version is now set correctly `Version 0.2.4 `_ ======================================================================================= From 5f810dcedb83369ca89c58cb22b3942bd943f049 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 16 Jul 2020 17:05:27 -0400 Subject: [PATCH 17/27] :books: Update usage to clarify / emphasize automatic & custom binding + remove optional `output_dir` (never used, overwritten by positional argument) --- src/cpac/__main__.py | 65 +++++++++++++++++++++++----------------- tests/test_cpac_utils.py | 2 +- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/cpac/__main__.py b/src/cpac/__main__.py index fbfac84..4aef76a 100644 --- a/src/cpac/__main__.py +++ b/src/cpac/__main__.py @@ -33,24 +33,35 @@ def parse_args(args): parser = argparse.ArgumentParser( description='cpac: a Python package that simplifies using C-PAC ' - ' containerized images. If no ' - 'platform nor image is specified, cpac will try Docker ' - 'first, then try Singularity if Docker fails.', - conflict_handler='resolve' + ' containerized images. \n\n' + 'This commandline interface package is designed to ' 'minimize repetition.\nAs such, nearly all arguments are ' + 'optional.\n\nWhen launching a container, this package will ' + 'try to bind any paths mentioned in \n • the command\n • ' + 'the data configuration\n\nAn example minimal run command ' + 'example:\n\tcpac run /path/to/data /path/for/outputs', + conflict_handler='resolve', + formatter_class=argparse.RawTextHelpFormatter ) - parser.add_argument('--platform', choices=['docker', 'singularity']) + parser.add_argument( + '--platform', + choices=['docker', 'singularity'], + help='If neither platform nor image is specified,\ncpac will try ' + 'Docker first, the try\nSingularity if Docker fails.' + ) parser.add_argument( '--image', help='path to Singularity image file OR name of Docker image (eg, ' - '"fcpindi/c-pac"). Will attempt to pull from Singularity Hub or ' - 'Docker Hub if not provided.' + '"fcpindi/c-pac").\nWill attempt to pull from Singularity Hub or ' + 'Docker Hub if not provided.\nIf image is specified but platform ' + 'is not, platform is\nassumed to be Singularity if image is a ' + 'path or \nDocker if image is an image name.' ) parser.add_argument( '--tag', - help='tag of the Docker image to use (eg, "latest" or "nightly"). ' + help='tag of the Docker image to use (eg, "latest" or "nightly").' ) parser.add_argument( @@ -80,40 +91,38 @@ def parse_args(args): parser.add_argument( '--working_dir', default=cwd, - help="working directory", - metavar="PATH" + help='working directory', + metavar='PATH' ) parser.add_argument( '--temp_dir', default='/tmp', - help="directory for temporary files", - metavar="PATH" - ) - - parser.add_argument( - '--output_dir', - default=os.path.join(cwd, 'outputs'), - help="directory where output files should be stored", - metavar="PATH" + help='directory for temporary files', + metavar='PATH' ) parser.add_argument( '-o', '--container_option', dest='container_option', - nargs='+', - help="parameters and flags to pass through to Docker or Singularity", - metavar="OPT" + nargs='*', + help='parameters and flags to pass through to Docker or Singularity\n' + '\nThis flag can take multiple arguments so cannot ' + 'be\nthe final argument before the command argument (i.e.,\nrun ' + 'or any other command that does not start with - or --)\n', + metavar='OPT' ) parser.add_argument( '-B', '--custom_binding', - dest="custom_binding", - nargs="+", - help="directory to bind to container with a different path than the " - "real path in the format real_path:container_path (eg, " - "/home/C-PAC/run5/outputs:/outputs). Use absolute paths for both " - "paths" + dest='custom_binding', + nargs='*', + help='directories to bind with a different path in\nthe container ' + 'than the real path of the directory.\nOne or more pairs in the ' 'format:\n\treal_path:container_path\n(eg, ' + '/home/C-PAC/run5/outputs:/outputs).\nUse absolute paths for ' + 'both paths.\n\nThis flag can take multiple arguments so cannot ' + 'be\nthe final argument before the command argument (i.e.,\nrun ' + 'or any other command that does not start with - or --)\n' ) subparsers = parser.add_subparsers(dest='command') diff --git a/tests/test_cpac_utils.py b/tests/test_cpac_utils.py index 1513ddf..3cd1e43 100644 --- a/tests/test_cpac_utils.py +++ b/tests/test_cpac_utils.py @@ -27,7 +27,7 @@ def test_utils_help(args, capsys, platform): def test_utils_new_settings_template(args, tmp_path): wd = tmp_path argv = ( - f'cpac {args} --working_dir {wd} --temp_dir {wd} --output_dir {wd} ' + f'cpac {args} --working_dir {wd} --temp_dir {wd} ' f'utils data_config new_settings_template' ).split(' ') with mock.patch.object(sys, 'argv', argv): From b73a6e792ec9e73668f9855a87eaad63d0385588 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 16 Jul 2020 17:36:37 -0400 Subject: [PATCH 18/27] =?UTF-8?q?=F0=9F=93=9A=20Update=20the=20main=20usag?= =?UTF-8?q?e=20string=20to=20better=20articulate=20functionality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a2aa180..ef22c5b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,18 +2,19 @@ Changelog ========= `Version 0.2.5 `_ -======================================================================================= +======================================================================================== +* 📚 Update the main usage string to better articulate functionality * 📢🐳 Provide a clearer error message if package cannot connect to Docker. * 🐳 Fix a bug introduced in `v0.2.4 ` where some crashfiles would print for ``cpac --platform singularity crash`` but not for ``cpac --platform docker crash`` `Version 0.2.4 `_ -======================================================================================= +======================================================================================== * 💪 Make ``crash`` command automatically touch (within a container) all missing files a crashfile requires to exist and print the underlying output * 🐳 Make Docker commands (especially ``pull`` and ``crash``) more robust * ⬆️ Require Python ≥3.6 (for fstrings) `Version 0.2.3: Crashfile `_ -================================================================================================== +===================================================================================================== * ✨ Added ``group`` and ``crash`` commands * 🚑 Fixed a bug where pass-through flags were being mangled * 🖇️ Binds any directories necessary to access any paths found in pass-through CLI arguments From b8414ca1c30cabd5e1bbb5b8487cbcc15e601fcb Mon Sep 17 00:00:00 2001 From: Travis Date: Sat, 18 Jul 2020 19:23:40 +0000 Subject: [PATCH 19/27] :books: Update usage from helpstring [skip travis] --- README.rst | 58 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 1b46167..5bdc215 100644 --- a/README.rst +++ b/README.rst @@ -30,13 +30,20 @@ Usage cpac --help usage: cpac [-h] [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] [--version] [-v] [-vv] [--working_dir PATH] [--temp_dir PATH] - [--output_dir PATH] [-o OPT [OPT ...]] - [-B CUSTOM_BINDING [CUSTOM_BINDING ...]] + [-o [OPT [OPT ...]]] [-B [CUSTOM_BINDING [CUSTOM_BINDING ...]]] {run,group,utils,crash} ... - cpac: a Python package that simplifies using C-PAC - containerized images. If no platform nor image is specified, cpac will try - Docker first, then try Singularity if Docker fails. + cpac: a Python package that simplifies using C-PAC containerized images. + + This commandline interface package is designed to minimize repetition. + As such, nearly all arguments are optional. + + When launching a container, this package will try to bind any paths mentioned in + • the command + • the data configuration + + An example minimal run command example: + cpac run /path/to/data /path/for/outputs positional arguments: {run,group,utils,crash} @@ -44,26 +51,37 @@ Usage optional arguments: -h, --help show this help message and exit --platform {docker,singularity} - --image IMAGE path to Singularity image file OR name of Docker image - (eg, "fcpindi/c-pac"). Will attempt to pull from - Singularity Hub or Docker Hub if not provided. - --tag TAG tag of the Docker image to use (eg, "latest" or - "nightly"). + If neither platform nor image is specified, + cpac will try Docker first, the try + Singularity if Docker fails. + --image IMAGE path to Singularity image file OR name of Docker image (eg, "fcpindi/c-pac"). + Will attempt to pull from Singularity Hub or Docker Hub if not provided. + If image is specified but platform is not, platform is + assumed to be Singularity if image is a path or + Docker if image is an image name. + --tag TAG tag of the Docker image to use (eg, "latest" or "nightly"). --version show program's version number and exit -v, --verbose set loglevel to INFO -vv, --very-verbose set loglevel to DEBUG --working_dir PATH working directory --temp_dir PATH directory for temporary files - --output_dir PATH directory where output files should be stored - -o OPT [OPT ...], --container_option OPT [OPT ...] - parameters and flags to pass through to Docker or - Singularity - -B CUSTOM_BINDING [CUSTOM_BINDING ...], --custom_binding CUSTOM_BINDING [CUSTOM_BINDING ...] - directory to bind to container with a different path - than the real path in the format - real_path:container_path (eg, - /home/C-PAC/run5/outputs:/outputs). Use absolute paths - for both paths + -o [OPT [OPT ...]], --container_option [OPT [OPT ...]] + parameters and flags to pass through to Docker or Singularity + + This flag can take multiple arguments so cannot be + the final argument before the command argument (i.e., + run or any other command that does not start with - or --) + -B [CUSTOM_BINDING [CUSTOM_BINDING ...]], --custom_binding [CUSTOM_BINDING [CUSTOM_BINDING ...]] + directories to bind with a different path in + the container than the real path of the directory. + One or more pairs in the format: + real_path:container_path + (eg, /home/C-PAC/run5/outputs:/outputs). + Use absolute paths for both paths. + + This flag can take multiple arguments so cannot be + the final argument before the command argument (i.e., + run or any other command that does not start with - or --) .. END USAGE From ce5faa61e1fa604ec84868a1826c2e160eefd6c0 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 20 Jul 2020 11:52:01 -0400 Subject: [PATCH 20/27] :octocat: Add badge for GitHub Action: upload Python package --- README.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 1b46167..5ba375a 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ -=============================================================== -C-PAC Python Package |github-version| |build-status| |coverage| -=============================================================== +======================================================================== +C-PAC Python Package |build-status| |github-version| |upload| |coverage| +======================================================================== + A Python package that wraps `C-PAC `_, enabling users to install cpac with `pip `_ and run from the command line. @@ -76,3 +77,7 @@ Usage .. |coverage| image:: https://coveralls.io/repos/github/shnizzedy/cpac-python-package/badge.svg?branch=trunk :target: https://coveralls.io/github/shnizzedy/cpac-python-package?branch=trunk :alt: coverage badge +.. |upload| image:: https://github.com/shnizzedy/cpac-python-package/workflows/Upload%20Python%20Package/badge.svg + :target: https://pypi.org/project/cpac-py/ + :alt: upload Python package + From 1f02f4beb77b81025d5ee878541e8eabf827f7ad Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Tue, 25 Sep 2018 12:52:31 -0400 Subject: [PATCH 21/27] coverage report should use package relative path instead of absolute path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 22bc509..4cdb84a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ install: - source tests/travis_install.sh - pip install -r requirements.txt - pip install twine - - pip install . + - pip install -e . before_script: - git config --global user.email "support@travis-ci.org" - git config --global user.name "Travis" From 03f20476632a017b92f61218b6ab595ae5c2baf7 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 20 Jul 2020 13:41:59 -0400 Subject: [PATCH 22/27] =?UTF-8?q?=F0=9F=94=AC=20Set=20`coverage=20reports?= =?UTF-8?q?=20`?= =?UTF-8?q?=5F=20to=20report=20local=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a2aa180..76bce52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,8 @@ Changelog `Version 0.2.5 `_ ======================================================================================= * 📢🐳 Provide a clearer error message if package cannot connect to Docker. -* 🐳 Fix a bug introduced in `v0.2.4 ` where some crashfiles would print for ``cpac --platform singularity crash`` but not for ``cpac --platform docker crash`` +* 🐳 Fix a bug introduced in `v0.2.4 `_ where some crashfiles would print for ``cpac --platform singularity crash`` but not for ``cpac --platform docker crash`` +* 🔬 Set `coverage reports `_ to report local paths `Version 0.2.4 `_ ======================================================================================= From aaa4c21675e9683f8a74fad4bb10ef5242f4e7a3 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 21 Jul 2020 17:08:08 -0400 Subject: [PATCH 23/27] :ok_hand: Add example with optional arguments --- src/cpac/__main__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/cpac/__main__.py b/src/cpac/__main__.py index 4aef76a..246e7d3 100644 --- a/src/cpac/__main__.py +++ b/src/cpac/__main__.py @@ -35,10 +35,18 @@ def parse_args(args): description='cpac: a Python package that simplifies using C-PAC ' ' containerized images. \n\n' 'This commandline interface package is designed to ' 'minimize repetition.\nAs such, nearly all arguments are ' - 'optional.\n\nWhen launching a container, this package will ' - 'try to bind any paths mentioned in \n • the command\n • ' - 'the data configuration\n\nAn example minimal run command ' - 'example:\n\tcpac run /path/to/data /path/for/outputs', + 'optional.\n\nWhen launching a container, this package ' + 'will try to bind any paths mentioned in \n • the command' + '\n • the data configuration\n\nAn example minimal run ' + 'command:\n\tcpac run /path/to/data /path/for/outputs' + '\n\nAn example run command with optional arguments:\n\t' + 'cpac -B /path/to/data/configs:/configs \\\n\t\t' + '--image fcpindi/c-pac --tag latest \\\n\t\t' + 'run /path/to/data /path/for/outputs \\\n\t\t' + '--data_config_file /configs/data_config.yml \\\n\t\t' + '--save_working_dir\n\n' + 'Each command can take "--help" to provide additonal ' + 'usage information, e.g.,\n\n\tcpac run --help', conflict_handler='resolve', formatter_class=argparse.RawTextHelpFormatter ) From caf2760cadc31a87a473c466ec5a7f6b2fa7a2bd Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 21 Jul 2020 17:28:06 -0400 Subject: [PATCH 24/27] :children_crossing: Reorder usage options --- src/cpac/__main__.py | 80 ++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/cpac/__main__.py b/src/cpac/__main__.py index 246e7d3..ede73f4 100644 --- a/src/cpac/__main__.py +++ b/src/cpac/__main__.py @@ -50,6 +50,35 @@ def parse_args(args): conflict_handler='resolve', formatter_class=argparse.RawTextHelpFormatter ) + + parser.add_argument( + '--version', + action='version', + version='cpac {ver}'.format(ver=__version__) + ) + + parser.add_argument( + '-o', '--container_option', + dest='container_option', + nargs='*', + help='parameters and flags to pass through to Docker or Singularity\n' + '\nThis flag can take multiple arguments so cannot ' + 'be\nthe final argument before the command argument (i.e.,\nrun ' + 'or any other command that does not start with - or --)\n', + metavar='OPT' + ) + + parser.add_argument( + '-B', '--custom_binding', + dest='custom_binding', + nargs='*', + help='directories to bind with a different path in\nthe container ' + 'than the real path of the directory.\nOne or more pairs in the ' 'format:\n\treal_path:container_path\n(eg, ' + '/home/C-PAC/run5/outputs:/outputs).\nUse absolute paths for ' + 'both paths.\n\nThis flag can take multiple arguments so cannot ' + 'be\nthe final argument before the command argument (i.e.,\nrun ' + 'or any other command that does not start with - or --)\n' + ) parser.add_argument( '--platform', @@ -73,9 +102,17 @@ def parse_args(args): ) parser.add_argument( - '--version', - action='version', - version='cpac {ver}'.format(ver=__version__) + '--working_dir', + default=cwd, + help='working directory', + metavar='PATH' + ) + + parser.add_argument( + '--temp_dir', + default='/tmp', + help='directory for temporary files', + metavar='PATH' ) parser.add_argument( @@ -96,43 +133,6 @@ def parse_args(args): const=logging.DEBUG ) - parser.add_argument( - '--working_dir', - default=cwd, - help='working directory', - metavar='PATH' - ) - - parser.add_argument( - '--temp_dir', - default='/tmp', - help='directory for temporary files', - metavar='PATH' - ) - - parser.add_argument( - '-o', '--container_option', - dest='container_option', - nargs='*', - help='parameters and flags to pass through to Docker or Singularity\n' - '\nThis flag can take multiple arguments so cannot ' - 'be\nthe final argument before the command argument (i.e.,\nrun ' - 'or any other command that does not start with - or --)\n', - metavar='OPT' - ) - - parser.add_argument( - '-B', '--custom_binding', - dest='custom_binding', - nargs='*', - help='directories to bind with a different path in\nthe container ' - 'than the real path of the directory.\nOne or more pairs in the ' 'format:\n\treal_path:container_path\n(eg, ' - '/home/C-PAC/run5/outputs:/outputs).\nUse absolute paths for ' - 'both paths.\n\nThis flag can take multiple arguments so cannot ' - 'be\nthe final argument before the command argument (i.e.,\nrun ' - 'or any other command that does not start with - or --)\n' - ) - subparsers = parser.add_subparsers(dest='command') run_parser = subparsers.add_parser( From 8aa2ae1ce6440b35230d3af11aeb33bb12c8a80c Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 21 Jul 2020 17:28:39 -0400 Subject: [PATCH 25/27] :ok_hand: Fix typo --- src/cpac/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpac/__main__.py b/src/cpac/__main__.py index ede73f4..e9ae4c0 100644 --- a/src/cpac/__main__.py +++ b/src/cpac/__main__.py @@ -84,7 +84,7 @@ def parse_args(args): '--platform', choices=['docker', 'singularity'], help='If neither platform nor image is specified,\ncpac will try ' - 'Docker first, the try\nSingularity if Docker fails.' + 'Docker first, then try\nSingularity if Docker fails.' ) parser.add_argument( From 74026c3a9da9768b051f35a0ef17c4db36bcacb6 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 21 Jul 2020 21:42:38 +0000 Subject: [PATCH 26/27] :books: Update usage from helpstring --- README.rst | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index a4380e0..31d46a5 100644 --- a/README.rst +++ b/README.rst @@ -29,9 +29,10 @@ Usage .. code-block:: shell cpac --help - usage: cpac [-h] [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] - [--version] [-v] [-vv] [--working_dir PATH] [--temp_dir PATH] - [-o [OPT [OPT ...]]] [-B [CUSTOM_BINDING [CUSTOM_BINDING ...]]] + usage: cpac [-h] [--version] [-o [OPT [OPT ...]]] + [-B [CUSTOM_BINDING [CUSTOM_BINDING ...]]] + [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] + [--working_dir PATH] [--temp_dir PATH] [-v] [-vv] {run,group,utils,crash} ... cpac: a Python package that simplifies using C-PAC containerized images. @@ -43,29 +44,26 @@ Usage • the command • the data configuration - An example minimal run command example: + An example minimal run command: cpac run /path/to/data /path/for/outputs + An example run command with optional arguments: + cpac -B /path/to/data/configs:/configs \ + --image fcpindi/c-pac --tag latest \ + run /path/to/data /path/for/outputs \ + --data_config_file /configs/data_config.yml \ + --save_working_dir + + Each command can take "--help" to provide additonal usage information, e.g., + + cpac run --help + positional arguments: {run,group,utils,crash} optional arguments: -h, --help show this help message and exit - --platform {docker,singularity} - If neither platform nor image is specified, - cpac will try Docker first, the try - Singularity if Docker fails. - --image IMAGE path to Singularity image file OR name of Docker image (eg, "fcpindi/c-pac"). - Will attempt to pull from Singularity Hub or Docker Hub if not provided. - If image is specified but platform is not, platform is - assumed to be Singularity if image is a path or - Docker if image is an image name. - --tag TAG tag of the Docker image to use (eg, "latest" or "nightly"). --version show program's version number and exit - -v, --verbose set loglevel to INFO - -vv, --very-verbose set loglevel to DEBUG - --working_dir PATH working directory - --temp_dir PATH directory for temporary files -o [OPT [OPT ...]], --container_option [OPT [OPT ...]] parameters and flags to pass through to Docker or Singularity @@ -83,6 +81,20 @@ Usage This flag can take multiple arguments so cannot be the final argument before the command argument (i.e., run or any other command that does not start with - or --) + --platform {docker,singularity} + If neither platform nor image is specified, + cpac will try Docker first, then try + Singularity if Docker fails. + --image IMAGE path to Singularity image file OR name of Docker image (eg, "fcpindi/c-pac"). + Will attempt to pull from Singularity Hub or Docker Hub if not provided. + If image is specified but platform is not, platform is + assumed to be Singularity if image is a path or + Docker if image is an image name. + --tag TAG tag of the Docker image to use (eg, "latest" or "nightly"). + --working_dir PATH working directory + --temp_dir PATH directory for temporary files + -v, --verbose set loglevel to INFO + -vv, --very-verbose set loglevel to DEBUG .. END USAGE From 80b8574f24c44c55a247070494c0f973c88b033e Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 22 Jul 2020 12:36:50 -0400 Subject: [PATCH 27/27] =?UTF-8?q?=F0=9F=A4=9D=20Merge=20branch=20trunk=20i?= =?UTF-8?q?nto=20develop=20(#48)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit