From 192eefc70fe727768f565ad72da14347a2e0c50e Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 21 Aug 2024 13:15:37 +1200 Subject: [PATCH 01/12] Added a rudimentary version of --migrate-pytest-hard --- nf_core/__main__.py | 417 ++++++++++++++++-- nf_core/commands_modules.py | 37 +- nf_core/components/create.py | 142 +++++- nf_core/module-template/tests/main.nf.test.j2 | 10 + nf_core/modules/create.py | 2 + 5 files changed, 553 insertions(+), 55 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0efea13ec9..053a146bdf 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -54,7 +54,12 @@ ) from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE from nf_core.pipelines.download import DownloadError -from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir +from nf_core.utils import ( + check_if_outdated, + nfcore_logo, + rich_force_colors, + setup_nfcore_dir, +) # Set up logging as the root logger # Submodules should all traverse back to this @@ -85,7 +90,14 @@ }, { "name": "For developers", - "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"], + "commands": [ + "create", + "lint", + "bump-version", + "sync", + "schema", + "create-logo", + ], }, ], "nf-core modules": [ @@ -257,7 +269,13 @@ def pipelines(ctx): @click.option("-d", "--description", type=str, help="A short description of your pipeline") @click.option("-a", "--author", type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0.0dev", help="The initial version number to use") -@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") +@click.option( + "-f", + "--force", + is_flag=True, + default=False, + help="Overwrite output directory if it already exists", +) @click.option("-o", "--outdir", help="Output directory for new pipeline (default: pipeline name)") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") @click.option( @@ -270,7 +288,17 @@ def command_pipelines_create(ctx, name, description, author, version, force, out """ Create a new pipeline using the nf-core template. """ - pipelines_create(ctx, name, description, author, version, force, outdir, template_yaml, organisation) + pipelines_create( + ctx, + name, + description, + author, + version, + force, + outdir, + template_yaml, + organisation, + ) # nf-core pipelines lint @@ -346,7 +374,19 @@ def command_pipelines_lint( """ Check pipeline code against nf-core guidelines. """ - pipelines_lint(ctx, directory, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, sort_by) + pipelines_lint( + ctx, + directory, + release, + fix, + key, + show_passed, + fail_ignored, + fail_warned, + markdown, + json, + sort_by, + ) # nf-core pipelines download @@ -556,7 +596,18 @@ def command_pipelines_launch( """ Launch a pipeline using a web GUI or command line prompts. """ - pipelines_launch(ctx, pipeline, id, revision, command_only, params_in, params_out, save_all, show_hidden, url) + pipelines_launch( + ctx, + pipeline, + id, + revision, + command_only, + params_in, + params_out, + save_all, + show_hidden, + url, + ) # nf-core pipelines list @@ -613,12 +664,28 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived): @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") def command_pipelines_sync( - ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr + ctx, + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, ): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. """ - pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr) + pipelines_sync( + ctx, + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, + ) # nf-core pipelines bump-version @@ -650,7 +717,14 @@ def command_pipelines_bump_version(ctx, new_version, directory, nextflow): # nf-core pipelines create-logo @pipelines.command("create-logo") @click.argument("logo-text", metavar="") -@click.option("-d", "--dir", "directory", type=click.Path(), default=".", help="Directory to save the logo in.") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(), + default=".", + help="Directory to save the logo in.", +) @click.option( "-n", "--name", @@ -885,7 +959,13 @@ def command_modules_list_local(ctx, keywords, json, directory): # pylint: disab # nf-core modules install @modules.command("install") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -919,7 +999,13 @@ def command_modules_install(ctx, tool, directory, prompt, force, sha): # nf-core modules update @modules.command("update") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -990,13 +1076,31 @@ def command_modules_update( """ Update DSL2 modules within a pipeline. """ - modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output) + modules_update( + ctx, + tool, + directory, + force, + prompt, + sha, + install_all, + preview, + save_diff, + update_deps, + limit_output, + ) # nf-core modules patch @modules.command("patch") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1016,7 +1120,13 @@ def command_modules_patch(ctx, tool, directory, remove): # nf-core modules remove @modules.command("remove") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1036,7 +1146,14 @@ def command_modules_remove(ctx, directory, tool): @modules.command("create") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") -@click.option("-d", "--dir", "directory", type=click.Path(exists=True), default=".", metavar="") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(exists=True), + default=".", + metavar="", +) @click.option( "-a", "--author", @@ -1099,6 +1216,12 @@ def command_modules_remove(ctx, directory, tool): default=False, help="Migrate a module with pytest tests to nf-test", ) +@click.option( + "--migrate-pytest-hard", + is_flag=True, + default=False, + help="Try hard when migrating pytest tests", +) def command_modules_create( ctx, tool, @@ -1112,6 +1235,7 @@ def command_modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ): """ Create a new DSL2 module from the nf-core template. @@ -1129,13 +1253,20 @@ def command_modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ) # nf-core modules test @modules.command("test") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-v", "--verbose", @@ -1178,19 +1309,52 @@ def command_modules_create( default=False, help="Migrate a module with pytest tests to nf-test", ) -def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose): +@click.option( + "--migrate-pytest-hard", + is_flag=True, + default=False, + help="Try hard when migrating pytest tests", +) +def command_modules_test( + ctx, + tool, + directory, + no_prompts, + update, + once, + profile, + migrate_pytest, + migrate_pytest_hard, + verbose, +): """ Run nf-test for a module. """ if verbose: ctx.obj["verbose"] = verbose - modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) + modules_test( + ctx, + tool, + directory, + no_prompts, + update, + once, + profile, + migrate_pytest, + migrate_pytest_hard, + ) # nf-core modules lint @modules.command("lint") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1231,17 +1395,47 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile is_flag=True, help="Fix the module version if a newer version is available", ) -def command_modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version): +def command_modules_lint( + ctx, + tool, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + fix_version, +): """ Lint one or more modules in a directory. """ - modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version) + modules_lint( + ctx, + tool, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + fix_version, + ) # nf-core modules info @modules.command("info") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1260,7 +1454,13 @@ def command_modules_info(ctx, tool, directory): # nf-core modules bump-versions @modules.command("bump-versions") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1321,7 +1521,14 @@ def subworkflows(ctx, git_remote, branch, no_pull): @subworkflows.command("create") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") -@click.option("-d", "--dir", "directory", type=click.Path(exists=True), default=".", metavar="") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(exists=True), + default=".", + metavar="", +) @click.option( "-a", "--author", @@ -1352,7 +1559,13 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force, migr # nf-core subworkflows test @subworkflows.command("test") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1440,7 +1653,13 @@ def command_subworkflows_list_local(ctx, keywords, json, directory): # pylint: # nf-core subworkflows lint @subworkflows.command("lint") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1480,13 +1699,30 @@ def command_subworkflows_lint(ctx, subworkflow, directory, registry, key, all, f """ Lint one or more subworkflows in a directory. """ - subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by) + subworkflows_lint( + ctx, + subworkflow, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + ) # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1505,7 +1741,13 @@ def command_subworkflows_info(ctx, subworkflow, directory): # nf-core subworkflows install @subworkflows.command("install") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1545,7 +1787,13 @@ def command_subworkflows_install(ctx, subworkflow, directory, prompt, force, sha # nf-core subworkflows remove @subworkflows.command("remove") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1564,7 +1812,13 @@ def command_subworkflows_remove(ctx, directory, subworkflow): # nf-core subworkflows update @subworkflows.command("update") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1643,7 +1897,17 @@ def command_subworkflows_update( Update DSL2 subworkflow within a pipeline. """ subworkflows_update( - ctx, subworkflow, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output + ctx, + subworkflow, + directory, + force, + prompt, + sha, + install_all, + preview, + save_diff, + update_deps, + limit_output, ) @@ -1772,7 +2036,14 @@ def command_schema_docs(schema_path, output, format, force, columns): # nf-core create-logo (deprecated) @nf_core_cli.command("create-logo", deprecated=True, hidden=True) @click.argument("logo-text", metavar="") -@click.option("-d", "--dir", "directory", type=click.Path(), default=".", help="Directory to save the logo in.") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(), + default=".", + help="Directory to save the logo in.", +) @click.option( "-n", "--name", @@ -1849,14 +2120,30 @@ def command_create_logo(logo_text, directory, name, theme, width, format, force) @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") -def command_sync(directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr): +def command_sync( + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, +): """ Use `nf-core pipelines sync` instead. """ log.warning( "The `[magenta]nf-core sync[/]` command is deprecated. Use `[magenta]nf-core pipelines sync[/]` instead." ) - pipelines_sync(directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr) + pipelines_sync( + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, + ) # nf-core bump-version (deprecated) @@ -1976,7 +2263,18 @@ def command_launch( log.warning( "The `[magenta]nf-core launch[/]` command is deprecated. Use `[magenta]nf-core pipelines launch[/]` instead." ) - pipelines_launch(ctx, pipeline, id, revision, command_only, params_in, params_out, save_all, show_hidden, url) + pipelines_launch( + ctx, + pipeline, + id, + revision, + command_only, + params_in, + params_out, + save_all, + show_hidden, + url, + ) # nf-core create-params-file (deprecated) @@ -2202,7 +2500,19 @@ def command_lint( log.warning( "The `[magenta]nf-core lint[/]` command is deprecated. Use `[magenta]nf-core pipelines lint[/]` instead." ) - pipelines_lint(ctx, directory, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, sort_by) + pipelines_lint( + ctx, + directory, + release, + fix, + key, + show_passed, + fail_ignored, + fail_warned, + markdown, + json, + sort_by, + ) # nf-core create (deprecated) @@ -2216,7 +2526,13 @@ def command_lint( @click.option("-d", "--description", type=str, help="A short description of your pipeline") @click.option("-a", "--author", type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0.0dev", help="The initial version number to use") -@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") +@click.option( + "-f", + "--force", + is_flag=True, + default=False, + help="Overwrite output directory if it already exists", +) @click.option("-o", "--outdir", help="Output directory for new pipeline (default: pipeline name)") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") @click.option("--plain", is_flag=True, help="Use the standard nf-core template") @@ -2227,14 +2543,35 @@ def command_lint( help="The name of the GitHub organisation where the pipeline will be hosted (default: nf-core)", ) @click.pass_context -def command_create(ctx, name, description, author, version, force, outdir, template_yaml, plain, organisation): +def command_create( + ctx, + name, + description, + author, + version, + force, + outdir, + template_yaml, + plain, + organisation, +): """ Use `nf-core pipelines create` instead. """ log.warning( "The `[magenta]nf-core create[/]` command is deprecated. Use `[magenta]nf-core pipelines create[/]` instead." ) - pipelines_create(ctx, name, description, author, version, force, outdir, template_yaml, organisation) + pipelines_create( + ctx, + name, + description, + author, + version, + force, + outdir, + template_yaml, + organisation, + ) # Main script is being run - launch the CLI diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 57c8e9777c..30e8137cfe 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -175,6 +175,7 @@ def modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ): """ Create a new DSL2 module from the nf-core template. @@ -194,6 +195,10 @@ def modules_create( elif no_meta: has_meta = False + if migrate_pytest_hard and not migrate_pytest: + log.error("--migrate_pytest_hard can only allowed in combination with --migrate_pytest.") + sys.exit(1) + from nf_core.modules.create import ModuleCreate # Run function @@ -209,6 +214,7 @@ def modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ) module_create.create() except UserWarning as e: @@ -219,7 +225,17 @@ def modules_create( sys.exit(1) -def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest): +def modules_test( + ctx, + tool, + directory, + no_prompts, + update, + once, + profile, + migrate_pytest, + migrate_pytest_hard, +): """ Run nf-test for a module. @@ -227,6 +243,10 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat """ from nf_core.components.components_test import ComponentsTest + if migrate_pytest_hard and not migrate_pytest: + log.error("--migrate_pytest_hard can only allowed in combination with --migrate_pytest.") + sys.exit(1) + if migrate_pytest: modules_create( ctx, @@ -241,6 +261,7 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat conda_package_version=None, empty_template=False, migrate_pytest=migrate_pytest, + migrate_pytest_hard=migrate_pytest_hard, ) try: module_tester = ComponentsTest( @@ -261,7 +282,19 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat sys.exit(1) -def modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version): +def modules_lint( + ctx, + tool, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + fix_version, +): """ Lint one or more modules in a directory. diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c71b128415..72413858a2 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -40,6 +40,7 @@ def __init__( conda_version: Optional[str] = None, empty_template: bool = False, migrate_pytest: bool = False, + migrate_pytest_hard=False, ): super().__init__(component_type, directory) self.directory = directory @@ -61,6 +62,9 @@ def __init__( self.file_paths: Dict[str, Path] = {} self.not_empty_template = not empty_template self.migrate_pytest = migrate_pytest + self.migrate_pytest_hard = migrate_pytest_hard + self.pytest_units_str = None + self.pytest_has_nextflow_config = False def create(self) -> bool: """ @@ -160,12 +164,22 @@ def create(self) -> bool: not_alphabet = re.compile(r"[^a-zA-Z]") self.org_alphabet = not_alphabet.sub("", self.org) + # Extract pytest nextflow config + pytest_nextflow_config_contents = None + if self.migrate_pytest: + pytest_nextflow_config_contents = self._extract_nextflow_config() + + # Extract pytest units + if self.migrate_pytest and self.migrate_pytest_hard: + self._extract_pytest_units() + # Create component template with jinja2 assert self._render_template() log.info(f"Created component template: '{self.component_name}'") if self.migrate_pytest: self._copy_old_files(component_old_path) + self._copy_nextflow_config(pytest_nextflow_config_contents) log.info("Migrate pytest tests: Copied original module files to new module") shutil.rmtree(component_old_path) self._print_and_delete_pytest_files() @@ -458,6 +472,8 @@ def _copy_old_files(self, component_old_path): component_old_path / "templates", self.file_paths["environment.yml"].parent / "templates", ) + + def _extract_nextflow_config(self): # Create a nextflow.config file if it contains information other than publishDir pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) nextflow_config = pytest_dir / "nextflow.config" @@ -469,19 +485,28 @@ def _copy_old_files(self, component_old_path): config_lines += line # if the nextflow.config file only contained publishDir, non_publish_dir_lines will be 11 characters long (`process {\n}`) if len(config_lines) > 11: - log.debug("Copying nextflow.config file from pytest tests") - with open( - Path( - self.directory, - self.component_type, - self.org, - self.component_dir, - "tests", - "nextflow.config", - ), - "w+", - ) as ofh: - ofh.write(config_lines) + self.pytest_has_nextflow_config = True + return config_lines + + return None + + def _copy_nextflow_config(self, config_lines): + if config_lines is None: + return + + log.debug("Copying nextflow.config file from pytest tests") + with open( + Path( + self.directory, + self.component_type, + self.org, + self.component_dir, + "tests", + "nextflow.config", + ), + "w+", + ) as ofh: + ofh.write(config_lines) def _print_and_delete_pytest_files(self): """Prompt if pytest files should be deleted and printed to stdout""" @@ -519,3 +544,94 @@ def _print_and_delete_pytest_files(self): with open(modules_yml, "w") as fh: yaml.dump(yml_file, fh) run_prettier_on_file(modules_yml) + + def _extract_pytest_units(self): + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + main_nf_contents = Path(pytest_dir, "main.nf").read_text(encoding="UTF-8") + + main_nf_workflows = re.findall(r"workflow\s*(\w+)\s*{([^}]+)}", main_nf_contents, re.DOTALL) + + log.debug(f"Found {len(main_nf_workflows)} workflows {[x[0] for x in main_nf_workflows]}") + + nf_test_workflow = [] + for workflow in main_nf_workflows: + workflow_name = workflow[0] + workflow_content = str(workflow[1]) + + invoked_components = re.findall(r"(\w+)\s*\(([^\)]+)\)", workflow_content, re.DOTALL) + + invoked_components = [c for c in invoked_components if c[0] != "file"] + + log.debug(f"Found {len(invoked_components)} components invoked by {workflow_name}: {invoked_components}") + + if len(invoked_components) > 1: + raise ValueError( + f"Test workflow {workflow_name} invokes multiple components. This is not supported currently." + ) + + # TODO: Generalize to multiple components + invoked_component = invoked_components[0] + + invoked_component_name = str(invoked_component[0]).strip() + invoked_component_args = str(invoked_component[1]).strip().split(",") + + if len(invoked_component_args) > 1: + raise ValueError( + f"Test workflow {workflow_name} has invoked a component {invoked_component_name} with multiple args {invoked_component_args}. This is not supported currently." + ) + + # TODO: Generalize to multiple args + invoked_component_arg = invoked_component_args[0] + + log.debug( + f"Looking for arg {invoked_component_arg} for {invoked_component_name} in workflow {workflow_name}" + ) + + if invoked_component_arg != "input": + raise ValueError( + f"Test workflow {workflow_name} has invoked a component {invoked_component_name} with arg {invoked_component_arg} other than 'input'. This is not supported currently." + ) + + arg_data = re.findall(r"input\s*=\s*(\[.*?\n\s*\])", workflow_content, re.DOTALL) + + log.debug(f"For arg {invoked_component_arg} found data: {arg_data}") + + if len(arg_data) > 1 or len(arg_data) == 0: + raise ValueError(f"{invoked_component_arg} data could not be parsed") + + arg_data = arg_data[0] + + nf_test_workflow.append({"name": workflow_name.replace("_", "-"), "input": arg_data}) + + log.debug(f"Scaffolding {len(nf_test_workflow)} nf-test(s)") + + test_units_str = "" + for test in nf_test_workflow: + input_data_lines = str(test["input"]).split("\n") + input_data_indented = ( + input_data_lines[0].strip() + + "\n" + + "\n".join(["\t\t\t\t" + line.strip() for line in input_data_lines[1:]]) + ) + test_unit_str = f""" + test("{test['name']}") {{ + + when {{ + process {{ + \"\"\" + input[0] = {input_data_indented} + \"\"\" + }} + }} + + then {{ + assertAll( + {{ assert process.success }}, + {{ assert snapshot(process.out).match() }} + ) + }} + }} + """ + test_units_str += test_unit_str + + self.pytest_units_str = test_units_str diff --git a/nf_core/module-template/tests/main.nf.test.j2 b/nf_core/module-template/tests/main.nf.test.j2 index a50ecc6a07..c29585b52b 100644 --- a/nf_core/module-template/tests/main.nf.test.j2 +++ b/nf_core/module-template/tests/main.nf.test.j2 @@ -1,10 +1,15 @@ +{%- if not migrate_pytest_hard %} // TODO nf-core: Once you have added the required tests, please run the following command to build this file: // nf-core modules test {{ component_name }} +{%- endif %} nextflow_process { name "Test Process {{ component_name_underscore|upper }}" script "../main.nf" process "{{ component_name_underscore|upper }}" + {%- if pytest_has_nextflow_config %} + config "./nextflow.config" + {%- endif %} tag "modules" tag "modules_{{ org_alphabet }}" @@ -13,6 +18,7 @@ nextflow_process { {%- endif %} tag "{{ component_name }}" + {%- if not migrate_pytest_hard %} // TODO nf-core: Change the test name preferably indicating the test-data and file-format used test("sarscov2 - bam") { @@ -78,5 +84,9 @@ nextflow_process { } } + {%- endif %} +{%- if migrate_pytest_hard %} +{{ pytest_units_str }} +{%- endif %} } diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index a5e0795a9f..208b8bc667 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -18,6 +18,7 @@ def __init__( conda_version=None, empty_template=False, migrate_pytest=False, + migrate_pytest_hard=False, ): super().__init__( "modules", @@ -31,4 +32,5 @@ def __init__( conda_version, empty_template, migrate_pytest, + migrate_pytest_hard, ) From edfb5969a2c7c8f653d3649af3c6eb90768826cd Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 21 Aug 2024 14:28:44 +1200 Subject: [PATCH 02/12] Can now handle multiple input args --- nf_core/components/create.py | 96 +++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 34 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 72413858a2..50675bc34b 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -573,53 +573,26 @@ def _extract_pytest_units(self): invoked_component = invoked_components[0] invoked_component_name = str(invoked_component[0]).strip() - invoked_component_args = str(invoked_component[1]).strip().split(",") + invoked_component_args = [arg.strip() for arg in str(invoked_component[1]).split(",")] - if len(invoked_component_args) > 1: - raise ValueError( - f"Test workflow {workflow_name} has invoked a component {invoked_component_name} with multiple args {invoked_component_args}. This is not supported currently." - ) - - # TODO: Generalize to multiple args - invoked_component_arg = invoked_component_args[0] - - log.debug( - f"Looking for arg {invoked_component_arg} for {invoked_component_name} in workflow {workflow_name}" + arg_data = self._extract_pytest_args_data( + workflow_name, workflow_content, invoked_component_name, invoked_component_args ) - if invoked_component_arg != "input": - raise ValueError( - f"Test workflow {workflow_name} has invoked a component {invoked_component_name} with arg {invoked_component_arg} other than 'input'. This is not supported currently." - ) - - arg_data = re.findall(r"input\s*=\s*(\[.*?\n\s*\])", workflow_content, re.DOTALL) - - log.debug(f"For arg {invoked_component_arg} found data: {arg_data}") - - if len(arg_data) > 1 or len(arg_data) == 0: - raise ValueError(f"{invoked_component_arg} data could not be parsed") - - arg_data = arg_data[0] - nf_test_workflow.append({"name": workflow_name.replace("_", "-"), "input": arg_data}) - log.debug(f"Scaffolding {len(nf_test_workflow)} nf-test(s)") - test_units_str = "" for test in nf_test_workflow: - input_data_lines = str(test["input"]).split("\n") - input_data_indented = ( - input_data_lines[0].strip() - + "\n" - + "\n".join(["\t\t\t\t" + line.strip() for line in input_data_lines[1:]]) - ) + log.debug(f"Scaffolding nf-test '{test['name']}'") + + input_data_lines = self._make_nf_test_input(test["input"]) test_unit_str = f""" test("{test['name']}") {{ when {{ process {{ \"\"\" - input[0] = {input_data_indented} + {input_data_lines} \"\"\" }} }} @@ -635,3 +608,58 @@ def _extract_pytest_units(self): test_units_str += test_unit_str self.pytest_units_str = test_units_str + + def _extract_pytest_args_data( + self, workflow_name, workflow_content, invoked_component_name, invoked_component_args + ) -> list[str]: + return [ + self._extract_pytest_arg_data(workflow_name, workflow_content, invoked_component_name, arg) + for arg in invoked_component_args + ] + + def _extract_pytest_arg_data(self, workflow_name, workflow_content, invoked_component_name, invoked_component_arg): + log.debug( + f"Looking for arg '{invoked_component_arg}' for '{invoked_component_name}' in workflow '{workflow_name}'" + ) + + re_matches = self._extract_pytest_arg_matches(invoked_component_arg, workflow_content) + + if len(re_matches) != 1: + raise ValueError(f"'{invoked_component_arg}' data could not be parsed from matches {re_matches}") + + found_arg_data = re_matches[0] + + log.debug(f"For arg '{invoked_component_arg}' found data {found_arg_data}") + + return found_arg_data + + def _extract_pytest_arg_matches(self, invoked_component_arg, workflow_content): + # list such as input = [etc] + list_match = re.findall(rf"{invoked_component_arg}\s*=\s*(\[.*?\n\s*\])", workflow_content, re.DOTALL) + + if list_match != []: + return list_match + + # String match such as 'etc', "etc" + return re.findall(rf"{invoked_component_arg}\s*=\s*(['\"]+.*?['\"]+)", workflow_content, re.DOTALL) + + def _make_nf_test_input(self, input_data): + input_data_lines = "" + for index in range(len(input_data)): + arg_data = input_data[index] + if "\n" in arg_data: + input_data_str = self._indent_nf_test_arg(arg_data) + else: + input_data_str = arg_data.strip() + + indent = "" + if index > 0: + indent = "\t\t\t\t" + input_data_lines += indent + f"input[{index}] = " + input_data_str + "\n" + + return input_data_lines + + def _indent_nf_test_arg(self, arg_data): + arg_data_lines = arg_data.split("\n") + + return arg_data_lines[0].strip() + "\n" + "\n".join(["\t\t\t\t" + line.strip() for line in arg_data_lines[1:]]) From 3e3af6c03ab15d138c5a435fd8a5c200b3b7d7d2 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 21 Aug 2024 16:41:54 +1200 Subject: [PATCH 03/12] Now can handle [] as input and add stub option if stub in workflow name --- nf_core/components/create.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 50675bc34b..01be5e6b92 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -583,12 +583,14 @@ def _extract_pytest_units(self): test_units_str = "" for test in nf_test_workflow: - log.debug(f"Scaffolding nf-test '{test['name']}'") + test_name = test["name"] + log.debug(f"Scaffolding nf-test '{test_name}'") input_data_lines = self._make_nf_test_input(test["input"]) + add_stub_option = "options '-stub'" if "stub" in test_name else "" test_unit_str = f""" - test("{test['name']}") {{ - + test("{test_name}") {{ + {add_stub_option} when {{ process {{ \"\"\" @@ -618,6 +620,10 @@ def _extract_pytest_args_data( ] def _extract_pytest_arg_data(self, workflow_name, workflow_content, invoked_component_name, invoked_component_arg): + if "[" in invoked_component_arg: + log.debug(f"Arg '{invoked_component_arg}' is a value") + return invoked_component_arg + log.debug( f"Looking for arg '{invoked_component_arg}' for '{invoked_component_name}' in workflow '{workflow_name}'" ) @@ -634,9 +640,15 @@ def _extract_pytest_arg_data(self, workflow_name, workflow_content, invoked_comp return found_arg_data def _extract_pytest_arg_matches(self, invoked_component_arg, workflow_content): - # list such as input = [etc] + # multiline list such as input = [etc] list_match = re.findall(rf"{invoked_component_arg}\s*=\s*(\[.*?\n\s*\])", workflow_content, re.DOTALL) + if list_match != []: + return list_match + + # simple list such as input = [ ] + list_match = re.findall(rf"{invoked_component_arg}\s*=\s*(\[\s*\])", workflow_content, re.DOTALL) + if list_match != []: return list_match From 52564cb903018e8db25fa93058532284fea94a65 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 21 Aug 2024 17:46:19 +1200 Subject: [PATCH 04/12] Can handle [[],[]] inputs now --- nf_core/components/create.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 01be5e6b92..c6bbad7b65 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -573,7 +573,7 @@ def _extract_pytest_units(self): invoked_component = invoked_components[0] invoked_component_name = str(invoked_component[0]).strip() - invoked_component_args = [arg.strip() for arg in str(invoked_component[1]).split(",")] + invoked_component_args = self._split_pytest_component_args(invoked_component[1].strip()) arg_data = self._extract_pytest_args_data( workflow_name, workflow_content, invoked_component_name, invoked_component_args @@ -675,3 +675,19 @@ def _indent_nf_test_arg(self, arg_data): arg_data_lines = arg_data.split("\n") return arg_data_lines[0].strip() + "\n" + "\n".join(["\t\t\t\t" + line.strip() for line in arg_data_lines[1:]]) + + def _split_pytest_component_args(self, args_str: str) -> list[str]: + # Single argument case + if "," not in args_str: + return [args_str.strip()] + + args = [] + + arg_matches = re.findall(r"(\w+\s*|\[\s*\]|\[\[\],\[\]\])", args_str) + + log.debug(f"For args string {args_str} found matches {arg_matches}") + + for arg_match in arg_matches: + args.append(str(arg_match).strip()) + + return args From e4964ced1478c066ef54c4b4387cc4d9ce3cb33a Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Thu, 22 Aug 2024 14:27:45 +1200 Subject: [PATCH 05/12] Aded a check for the number of include statements --- nf_core/components/create.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c6bbad7b65..3b70003b97 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -549,6 +549,13 @@ def _extract_pytest_units(self): pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) main_nf_contents = Path(pytest_dir, "main.nf").read_text(encoding="UTF-8") + include_statements = re.findall(r"include\s*{\s*(\w+)\s*as?\s*(\w+)?\s*}", main_nf_contents) + + log.debug(f"Found {len(include_statements)} include statements {include_statements}") + + if len(include_statements) > 1: + raise ValueError("Multiple include statements are not yet supported") + main_nf_workflows = re.findall(r"workflow\s*(\w+)\s*{([^}]+)}", main_nf_contents, re.DOTALL) log.debug(f"Found {len(main_nf_workflows)} workflows {[x[0] for x in main_nf_workflows]}") From ab9b9ae00ce89476e27882b5c2294b377f69843e Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Thu, 22 Aug 2024 16:41:06 +1200 Subject: [PATCH 06/12] Now using symbol substitution to handle cases such as [[], xyz] --- nf_core/components/create.py | 128 ++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 46 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 3b70003b97..07805e3a81 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -484,7 +484,7 @@ def _extract_nextflow_config(self): if "publishDir" not in line and line.strip() != "": config_lines += line # if the nextflow.config file only contained publishDir, non_publish_dir_lines will be 11 characters long (`process {\n}`) - if len(config_lines) > 11: + if not re.match(r"^\s*process\s*{\s*}\s*$", config_lines, re.DOTALL): self.pytest_has_nextflow_config = True return config_lines @@ -549,7 +549,7 @@ def _extract_pytest_units(self): pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) main_nf_contents = Path(pytest_dir, "main.nf").read_text(encoding="UTF-8") - include_statements = re.findall(r"include\s*{\s*(\w+)\s*as?\s*(\w+)?\s*}", main_nf_contents) + include_statements = re.findall(r"include\s*{\s*(\w+)([\sas]+)?(\w+)?\s*}", main_nf_contents) log.debug(f"Found {len(include_statements)} include statements {include_statements}") @@ -565,11 +565,17 @@ def _extract_pytest_units(self): workflow_name = workflow[0] workflow_content = str(workflow[1]) + log.debug(f"Looking for NXF symbols in workflow {workflow_name}") + + nxf_symbols = self._extract_pytest_nxf_symbols(workflow_content) + invoked_components = re.findall(r"(\w+)\s*\(([^\)]+)\)", workflow_content, re.DOTALL) invoked_components = [c for c in invoked_components if c[0] != "file"] - log.debug(f"Found {len(invoked_components)} components invoked by {workflow_name}: {invoked_components}") + log.debug( + f"Found {len(invoked_components)} component(s) invoked by '{workflow_name}': {invoked_components}" + ) if len(invoked_components) > 1: raise ValueError( @@ -579,14 +585,12 @@ def _extract_pytest_units(self): # TODO: Generalize to multiple components invoked_component = invoked_components[0] - invoked_component_name = str(invoked_component[0]).strip() - invoked_component_args = self._split_pytest_component_args(invoked_component[1].strip()) + # invoked_component_name = invoked_component[0].strip() + invoked_component_args = invoked_component[1].strip() - arg_data = self._extract_pytest_args_data( - workflow_name, workflow_content, invoked_component_name, invoked_component_args - ) + extracted_component_args = self._extract_pytest_component_args(invoked_component_args, nxf_symbols) - nf_test_workflow.append({"name": workflow_name.replace("_", "-"), "input": arg_data}) + nf_test_workflow.append({"name": workflow_name.replace("_", "-"), "input": extracted_component_args}) test_units_str = "" for test in nf_test_workflow: @@ -618,49 +622,57 @@ def _extract_pytest_units(self): self.pytest_units_str = test_units_str - def _extract_pytest_args_data( - self, workflow_name, workflow_content, invoked_component_name, invoked_component_args - ) -> list[str]: - return [ - self._extract_pytest_arg_data(workflow_name, workflow_content, invoked_component_name, arg) - for arg in invoked_component_args - ] + def _extract_pytest_nxf_symbols(self, workflow_content: str) -> dict[str, str]: + symbols = {} + found_all = False - def _extract_pytest_arg_data(self, workflow_name, workflow_content, invoked_component_name, invoked_component_arg): - if "[" in invoked_component_arg: - log.debug(f"Arg '{invoked_component_arg}' is a value") - return invoked_component_arg + remaining_content = workflow_content + while not found_all: + match = self._extract_pytest_nxf_symbol_match(remaining_content) + if match: + match = match[0] - log.debug( - f"Looking for arg '{invoked_component_arg}' for '{invoked_component_name}' in workflow '{workflow_name}'" - ) + log.debug(f"Found symbol '{match[0]}' with data {match[1]}") + remaining_content = remaining_content.replace(f"{match[0]}={match[1]}", "", 1) + symbols[match[0].strip()] = match[1].strip() + continue - re_matches = self._extract_pytest_arg_matches(invoked_component_arg, workflow_content) + found_all = True - if len(re_matches) != 1: - raise ValueError(f"'{invoked_component_arg}' data could not be parsed from matches {re_matches}") + return symbols - found_arg_data = re_matches[0] - - log.debug(f"For arg '{invoked_component_arg}' found data {found_arg_data}") - - return found_arg_data - - def _extract_pytest_arg_matches(self, invoked_component_arg, workflow_content): + def _extract_pytest_nxf_symbol_match(self, workflow_content): # multiline list such as input = [etc] - list_match = re.findall(rf"{invoked_component_arg}\s*=\s*(\[.*?\n\s*\])", workflow_content, re.DOTALL) + list_match = re.findall(r"(\w+\s*)=(\s*\[.*?\n\s*\])", workflow_content, re.DOTALL) if list_match != []: return list_match # simple list such as input = [ ] - list_match = re.findall(rf"{invoked_component_arg}\s*=\s*(\[\s*\])", workflow_content, re.DOTALL) + list_match = re.findall(r"(\w+\s*)=(\s*\[\s*\])", workflow_content, re.DOTALL) if list_match != []: return list_match # String match such as 'etc', "etc" - return re.findall(rf"{invoked_component_arg}\s*=\s*(['\"]+.*?['\"]+)", workflow_content, re.DOTALL) + string_match = re.findall(r"(\w+\s*)=(\s*['\"][^'\"]*['\"])", workflow_content) + + if string_match != []: + return string_match + + # Number match such as 123.1 + num_match = re.findall(r"(\w+\s*)=(\s*[\d\.]+)", workflow_content, re.DOTALL) + + if num_match != []: + return num_match + + # File match such as file(params.test_data['sarscov2']['genome']['transcriptome_fasta'], checkIfExists: true) + file_match = re.findall(r"(\w+\s*)=(\s*file\s*\(.*?\s*\))", workflow_content, re.DOTALL) + + if file_match != []: + return file_match + + return [] def _make_nf_test_input(self, input_data): input_data_lines = "" @@ -683,18 +695,42 @@ def _indent_nf_test_arg(self, arg_data): return arg_data_lines[0].strip() + "\n" + "\n".join(["\t\t\t\t" + line.strip() for line in arg_data_lines[1:]]) - def _split_pytest_component_args(self, args_str: str) -> list[str]: - # Single argument case - if "," not in args_str: - return [args_str.strip()] + def _extract_pytest_component_args(self, args_str: str, nxf_symbols: dict[str, str]) -> list[str]: + # Single argument + if args_str in nxf_symbols.keys(): + return [nxf_symbols[args_str]] + + # All arguments are named + match = re.match(r"^[\sa-zA-Z_,]+$", args_str, re.DOTALL) + + if match: + return [nxf_symbols[arg.strip()] for arg in args_str.split(",")] + + # Split args while keeping brackets grouped + args = re.findall(r"\[.+\]|\w+|\[\]", args_str) + + if not args: + raise ValueError(f"Can not split args: {args_str}") + + log.debug(f"Split args: {args_str} into: {args}") + + # Replace the symbols embedded in list args + normalized_args = [] + for arg in args: + if "[" in arg: + normalized_arg = self._replace_pytest_symbol_in_list_arg(arg, nxf_symbols) + normalized_args.append(normalized_arg) + continue - args = [] + # For remaining args, try to replace symbol if possible + normalized_args.append(nxf_symbols[arg] if arg in nxf_symbols.keys() else arg) - arg_matches = re.findall(r"(\w+\s*|\[\s*\]|\[\[\],\[\]\])", args_str) + return normalized_args - log.debug(f"For args string {args_str} found matches {arg_matches}") + def _replace_pytest_symbol_in_list_arg(self, arg: str, nxf_symbols: dict[str, str]) -> str: + symbols = re.findall(r"[a-zA-Z_]+", arg) - for arg_match in arg_matches: - args.append(str(arg_match).strip()) + for symbol in symbols: + arg = arg.replace(symbol, nxf_symbols[symbol] if symbol in nxf_symbols.keys() else symbol, 1) - return args + return arg From dd7798992846945299b2916c8030fa8ed7436b0c Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Thu, 22 Aug 2024 16:59:05 +1200 Subject: [PATCH 07/12] Can now handle empty arg lists --- nf_core/components/create.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 07805e3a81..54c9463b8e 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -569,7 +569,7 @@ def _extract_pytest_units(self): nxf_symbols = self._extract_pytest_nxf_symbols(workflow_content) - invoked_components = re.findall(r"(\w+)\s*\(([^\)]+)\)", workflow_content, re.DOTALL) + invoked_components = re.findall(r"(\w+)\s*\(([^\)]*)\)", workflow_content, re.DOTALL) invoked_components = [c for c in invoked_components if c[0] != "file"] @@ -696,6 +696,10 @@ def _indent_nf_test_arg(self, arg_data): return arg_data_lines[0].strip() + "\n" + "\n".join(["\t\t\t\t" + line.strip() for line in arg_data_lines[1:]]) def _extract_pytest_component_args(self, args_str: str, nxf_symbols: dict[str, str]) -> list[str]: + # No arg + if args_str == "": + return [] + # Single argument if args_str in nxf_symbols.keys(): return [nxf_symbols[args_str]] From 0d5a416d0497aad6a80799b7eac9fae5b7c6b7e3 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Fri, 23 Aug 2024 10:24:08 +1200 Subject: [PATCH 08/12] Now power assertions are a bit automated --- nf_core/components/create.py | 63 ++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 54c9463b8e..2840f59895 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -141,6 +141,23 @@ def create(self) -> bool: # Check existence of directories early for fast-fail self.file_paths = self._get_component_dirs() + component_outputs = {} # output: meta_data + if self.migrate_pytest and self.migrate_pytest_hard: + # Extract outputs data + main_nf_data = Path( + self.directory, self.component_type, self.org, self.component_dir, "main.nf" + ).read_text() + if "output:" not in main_nf_data: + log.debug(f"Could not find any outputs in {self.component_name}") + else: + component_outputs_data = main_nf_data.split("output:")[1].split("when:")[0] + matches = re.findall(r"(.*)emit:\s*([^)\s,]+)", component_outputs_data, re.MULTILINE) + for match in matches: + component_outputs[match[1]] = match[0] + log.debug( + f"Found {len(component_outputs)} outputs in {self.component_name}: {component_outputs.keys()}" + ) + if self.migrate_pytest: # Rename the component directory to old component_old_dir = Path(str(self.component_dir) + "_old") @@ -171,7 +188,7 @@ def create(self) -> bool: # Extract pytest units if self.migrate_pytest and self.migrate_pytest_hard: - self._extract_pytest_units() + self._extract_pytest_units(component_outputs) # Create component template with jinja2 assert self._render_template() @@ -545,7 +562,7 @@ def _print_and_delete_pytest_files(self): yaml.dump(yml_file, fh) run_prettier_on_file(modules_yml) - def _extract_pytest_units(self): + def _extract_pytest_units(self, component_outputs): pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) main_nf_contents = Path(pytest_dir, "main.nf").read_text(encoding="UTF-8") @@ -599,6 +616,9 @@ def _extract_pytest_units(self): input_data_lines = self._make_nf_test_input(test["input"]) add_stub_option = "options '-stub'" if "stub" in test_name else "" + + power_assertions = self._get_power_assertions(component_outputs, is_stub=("stub" in test_name)) + test_unit_str = f""" test("{test_name}") {{ {add_stub_option} @@ -613,7 +633,7 @@ def _extract_pytest_units(self): then {{ assertAll( {{ assert process.success }}, - {{ assert snapshot(process.out).match() }} + {power_assertions} ) }} }} @@ -738,3 +758,40 @@ def _replace_pytest_symbol_in_list_arg(self, arg: str, nxf_symbols: dict[str, st arg = arg.replace(symbol, nxf_symbols[symbol] if symbol in nxf_symbols.keys() else symbol, 1) return arg + + def _get_power_assertions(self, component_outputs, is_stub): + power_assertions = "{ assert snapshot(process.out).match() }" + + if is_stub: + return power_assertions + + non_stable_outputs = ["bam", "txt", "log", "gz"] + + outputs_str = " ".join([f"{key} {value}" for (key, value) in component_outputs.items()]).lower() + has_non_stable = any([ns_output in outputs_str for ns_output in non_stable_outputs]) + + if not has_non_stable: + return power_assertions + + power_assertions = "{ assert snapshot(" + for output_name, output_meta in component_outputs.items(): + if output_name == "versions": + continue + + if "bam" in output_name or "bam" in output_meta: + power_assertions += f"\n\t\t\t\t\tbam(process.out.{output_name}[0][1]).getReadsMD5()," + continue + + if "log" in output_name or "log" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).name," + continue + + if "gz" in output_name or "gz" in output_meta: + power_assertions += f"\n\t\t\t\t\tpath(process.out.{output_name}[0][1]).linesGzip[3..7]," + continue + + power_assertions += f"\n\t\t\t\t\tprocess.out.{output_name}," + + power_assertions += "\n\t\t\t\t\tprocess.out.versions\n\t\t\t\t\t).match()\n\t\t\t\t}" + + return power_assertions From c8251ccdf400a069e5bc5b852702fde6e231baf9 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Fri, 23 Aug 2024 10:40:07 +1200 Subject: [PATCH 09/12] Improved list input regex --- nf_core/components/create.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 2840f59895..72e4a3e68a 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -665,6 +665,13 @@ def _extract_pytest_nxf_symbol_match(self, workflow_content): # multiline list such as input = [etc] list_match = re.findall(r"(\w+\s*)=(\s*\[.*?\n\s*\])", workflow_content, re.DOTALL) + if list_match != []: + return list_match + + # second rule for multiline list such as input = [etc] + # The first rule might not be needed! + list_match = re.findall(r"(\w+\s*)=(\s*\[[^=]+\]\s+)", workflow_content, re.DOTALL) + if list_match != []: return list_match From fdbbad0ed169dc23f0216b3c8b19e1b7e0cfc2ec Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Fri, 23 Aug 2024 11:46:05 +1200 Subject: [PATCH 10/12] Fixed an issue where a string argument was not correctly parsed --- nf_core/components/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 72e4a3e68a..17c9a644a6 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -738,7 +738,7 @@ def _extract_pytest_component_args(self, args_str: str, nxf_symbols: dict[str, s return [nxf_symbols[arg.strip()] for arg in args_str.split(",")] # Split args while keeping brackets grouped - args = re.findall(r"\[.+\]|\w+|\[\]", args_str) + args = re.findall(r"\[.+\]|\w+|\[\]|[\w'\"]+", args_str) if not args: raise ValueError(f"Can not split args: {args_str}") From 3060985707466fc4a528887c58b74b571529c179 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Fri, 23 Aug 2024 16:26:50 +1200 Subject: [PATCH 11/12] Took care of txt assertion and now false/true are not treated as nxf symbols --- nf_core/components/create.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 17c9a644a6..419cb205a5 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -735,7 +735,12 @@ def _extract_pytest_component_args(self, args_str: str, nxf_symbols: dict[str, s match = re.match(r"^[\sa-zA-Z_,]+$", args_str, re.DOTALL) if match: - return [nxf_symbols[arg.strip()] for arg in args_str.split(",")] + # Double check that any arg name is not on the prohibited list + args_list = args_str.split(",") + prohibited_names = ["false", "true"] + has_prohibited = any([arg.strip() in prohibited_names for arg in args_list]) + if not has_prohibited: + return [nxf_symbols[arg.strip()] for arg in args_str.split(",")] # Split args while keeping brackets grouped args = re.findall(r"\[.+\]|\w+|\[\]|[\w'\"]+", args_str) @@ -797,6 +802,10 @@ def _get_power_assertions(self, component_outputs, is_stub): power_assertions += f"\n\t\t\t\t\tpath(process.out.{output_name}[0][1]).linesGzip[3..7]," continue + if "txt" in output_name or "txt" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).readLines()[3..7]," + continue + power_assertions += f"\n\t\t\t\t\tprocess.out.{output_name}," power_assertions += "\n\t\t\t\t\tprocess.out.versions\n\t\t\t\t\t).match()\n\t\t\t\t}" From 6a92299a3a49723b83cee308f86136b5873e0bc0 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Fri, 23 Aug 2024 21:00:43 +1200 Subject: [PATCH 12/12] Added power assertions for rds and png --- nf_core/components/create.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 419cb205a5..7e33c93cc0 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -777,7 +777,7 @@ def _get_power_assertions(self, component_outputs, is_stub): if is_stub: return power_assertions - non_stable_outputs = ["bam", "txt", "log", "gz"] + non_stable_outputs = ["bam", "txt", "log", "gz", "rds", "png"] outputs_str = " ".join([f"{key} {value}" for (key, value) in component_outputs.items()]).lower() has_non_stable = any([ns_output in outputs_str for ns_output in non_stable_outputs]) @@ -806,6 +806,14 @@ def _get_power_assertions(self, component_outputs, is_stub): power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).readLines()[3..7]," continue + if "rds" in output_name or "rds" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).name," + continue + + if "png" in output_name or "png" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).name," + continue + power_assertions += f"\n\t\t\t\t\tprocess.out.{output_name}," power_assertions += "\n\t\t\t\t\tprocess.out.versions\n\t\t\t\t\t).match()\n\t\t\t\t}"