Initialise wf-amplicon from wf-template #380342b

epi2me-labs · Feb 13, 2023 · 6985e44 · 6985e44
commit 6985e44
Show file tree

Hide file tree

Showing 126 changed files with 4,650 additions and 0 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,75 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["bug", "triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+      value: "A bug happened!"
+    validations:
+      required: true
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating System
+      description: What operating system are you running?
+      options:
+        - Windows 10
+        - Windows 11
+        - macOS
+        - ubuntu 18.04
+        - ubuntu 20.04
+    validations:
+      required: true
+  - type: dropdown
+    id: execution
+    attributes:
+      label: Workflow Execution
+      description: Where are you running the workflow?
+      options:
+        - EPI2ME Labs desktop application
+        - Command line
+        - EPI2ME
+        - Other (please describe)
+    validations:
+      required: true
+  - type: input
+    id: labs-version
+    attributes:
+      label: Workflow Execution - EPI2ME Labs Versions
+      description: If you're running using EPI2ME Labs please provide the version and the environment version (Click the blue help icon bottom left and select ":bout")?
+    validations:
+      required: false
+  - type: dropdown
+    id: profile
+    attributes:
+      label: Workflow Execution - CLI Execution Profile
+      description: If you're using the CLI to run the workflow, what profile are you using?
+      options:
+        - Docker
+        - Singularity
+    validations:
+      required: false
+  - type: input
+    id: version
+    attributes:
+      label: Workflow Version
+      description: What version of the workflow are you running?
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+    validations:
+      required: true
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+    - name: Nanopore customer support
+      url: https://nanoporetech.com/contact
+      about: For general support, including bioinformatics questions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,7 @@
+nextflow
+.nextflow*
+template-workflow
+.*.swp
+.*.swo
+*.pyc
+*.pyo
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,173 @@
+# Include shared CI
+include:
+    - project: "epi2melabs/ci-templates"
+      file: "wf-containers.yaml"
+
+variables:
+    # We'll use the single-file case for these runs
+    NF_WORKFLOW_OPTS: "--fastq test_data/reads.fastq.gz"
+    CI_FLAVOUR: "new" # set to "classic" for old-style CI
+    TEST_CMD: "bash test/run_fastq_ingress_test.sh"
+    S3_TEST_DATA: "s3://ont-exd-int-s3-euwst1-epi2me-labs/wf-template/cw-1019/test_data"
+
+
+# Remove this block in downstream templates
+singularity-run:
+    tags: [] # no need for big ram
+# end
+
+
+docker-run:
+
+    # Remove this directive in downstream templates
+    tags: [] # no need for big ram
+
+    # Define a 1D job matrix to inject a variable named MATRIX_NAME into
+    #   the CI environment, we can use the value of MATRIX_NAME to determine
+    #   which options to apply as part of the rules block below
+    # NOTE There is a slightly cleaner way to define this matrix to include
+    #   the variables, but it is broken when using long strings! See CW-756
+    parallel:
+        matrix:
+            # TODO: do we really need to run all from s3?
+            - MATRIX_NAME: [
+                "single-file", "single-file-s3",
+                "case01", "case01-s3",
+                "case02", "case02-s3",
+                "case03", "case03-s3",
+                "case04", "case04-s3",
+                "case05", "case05-s3",
+                "case06", "case06-s3",
+                "case07", "case07-s3",
+                "case08", "case08-s3", case08-unclassified, case08-unclassified-s3,
+                "case09", "case09-s3",
+                "case10", "case10-s3",
+                "case11", "case11-s3",
+                ]
+    rules:
+        # NOTE As we're overriding the rules block for the included docker-run
+        #   we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run
+        #   being incorrectly scheduled for "detached merge request pipelines" etc.
+        - if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template")
+          when: never
+        - if: $MATRIX_NAME == "single-file"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/reads.fastq.gz"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/reads.fastq.gz $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "single-file-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/reads.fastq.gz"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/reads.fastq.gz $$PWD/$$CI_PROJECT_NAME
+
+        - if: $MATRIX_NAME == "case01"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case01"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case01 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case01-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case01"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case01 $$PWD/$$CI_PROJECT_NAME
+
+        - if: $MATRIX_NAME == "case02"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case02"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case02 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case02-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case02"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case02 $$PWD/$$CI_PROJECT_NAME
+
+        - if: $MATRIX_NAME == "case03"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case03"
+              ASSERT_NEXTFLOW_FAILURE: "1"
+              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain FASTQ files and sub-directories."
+        - if: $MATRIX_NAME == "case03-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case03"
+              ASSERT_NEXTFLOW_FAILURE: "1"
+              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain FASTQ files and sub-directories."
+
+        - if: $MATRIX_NAME == "case04"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case04"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case04 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case04-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case04"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case04 $$PWD/$$CI_PROJECT_NAME
+
+        - if: $MATRIX_NAME == "case05"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case05 --sample_sheet test_data/fastq_ingress/case05/sample_sheet.csv"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case05 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case05-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case05 --sample_sheet $S3_TEST_DATA/fastq_ingress/case05/sample_sheet.csv"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case05 $$PWD/$$CI_PROJECT_NAME $S3_TEST_DATA/fastq_ingress/case05/sample_sheet.csv
+
+        - if: $MATRIX_NAME == "case06"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case06 --sample_sheet test_data/fastq_ingress/case06/sample_sheet.csv"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case06 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case06-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case06 --sample_sheet $S3_TEST_DATA/fastq_ingress/case06/sample_sheet.csv"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case06 $$PWD/$$CI_PROJECT_NAME $S3_TEST_DATA/fastq_ingress/case06/sample_sheet.csv
+
+        - if: $MATRIX_NAME == "case07"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case07 --sample_sheet test_data/fastq_ingress/case07/sample_sheet.csv"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case07 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case07-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case07 --sample_sheet $S3_TEST_DATA/fastq_ingress/case07/sample_sheet.csv"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case07 $$PWD/$$CI_PROJECT_NAME $S3_TEST_DATA/fastq_ingress/case07/sample_sheet.csv
+
+        - if: $MATRIX_NAME == "case08"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case08"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case08-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case08"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case08-unclassified"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case08 --analyse_unclassified"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case08-unclassified-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case08 --analyse_unclassified"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
+
+        - if: $MATRIX_NAME == "case09"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case09"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case09 $$PWD/$$CI_PROJECT_NAME
+        - if: $MATRIX_NAME == "case09-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case09"
+              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case09 $$PWD/$$CI_PROJECT_NAME
+
+        - if: $MATRIX_NAME == "case10"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case10"
+              ASSERT_NEXTFLOW_FAILURE: "1"
+              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain more than one level of sub-directories with FASTQ files."
+        - if: $MATRIX_NAME == "case10-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case10"
+              ASSERT_NEXTFLOW_FAILURE: "1"
+              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain more than one level of sub-directories with FASTQ files."
+
+        - if: $MATRIX_NAME == "case11"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case11 --sample_sheet test_data/fastq_ingress/case11/sample_sheet.csv"
+              ASSERT_NEXTFLOW_FAILURE: "1"
+              ASSERT_NEXTFLOW_FAILURE_REXP: "Invalid sample sheet: values in 'alias' column not unique"
+        - if: $MATRIX_NAME == "case11-s3"
+          variables:
+              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case11 --sample_sheet $S3_TEST_DATA/fastq_ingress/case11/sample_sheet.csv"
+              ASSERT_NEXTFLOW_FAILURE: "1"
+              ASSERT_NEXTFLOW_FAILURE_REXP: "Invalid sample sheet: values in 'alias' column not unique"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,34 @@
+repos:
+  - repo: local
+    hooks:
+      - id: docs_schema
+        name: docs_schema
+        entry: parse_docs -p docs -e .md -s intro links -oj nextflow_schema.json
+        language: python
+        always_run: true
+        pass_filenames: false
+        additional_dependencies:
+          - epi2melabs
+      - id: docs_readme
+        name: docs_readme
+        entry: parse_docs -p docs -e .md -s header intro quickstart links -ot README.md
+        language: python
+        always_run: true
+        pass_filenames: false
+        additional_dependencies:
+          - epi2melabs
+  - repo: https://github.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-rst-docstrings
+          - flake8-docstrings
+          - flake8-import-order
+          - flake8-forbid-visual-indent
+          - pep8-naming
+          - flake8-no-types
+          - flake8-builtins
+          - flake8-absolute-import
+          - flake8-print
+        entry: flake8 bin --import-order-style google --statistics --max-line-length 88
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,80 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [unreleased]
+### Added
+- GitHub issues template.
+- Return of metadata with fastqingress.
+- Check of number of samples and barcoded directories.
+- Example of how to use the metadata from `fastqingress`.
+- Implemented `--version`
+- `fastcat_extra_args` option to `fastq_ingress` to pass arbitrary arguments to `fastcat` (defaults to empty string).
+- `fastcat_stats` option to `fastq_ingress` to force generation of `fastcat` stats even when the input is only a single file (default is false).
+### Changed
+- `fastq_ingress` now returns `[metamap, path-to-fastcat-seqs, path-to-fastcat-stats | null]`.
+- Bumped base container to v0.2.0.
+- Use groovy script to ping after workflow has run.
+- Removed sanitize fastq option.
+- fastq_ingress now removes unclassified read folders by default.
+- Workflow name and version is now more prominently displayed on start
+### Fixed
+- Output argument in Fastqingress homogenised.
+- Sanitize fastq intermittent null object error.
+- Add `*.pyc` and `*.pyo` ignores to wf-template .gitignore
+### Note
+- Bumped version to `v4` to align versioning with Launcher v4
+
+## [v0.2.0]
+### Added
+- default process label parameter
+- Added `params.wf.example_cmd` list to populate `--help`
+### Changed
+- Update WorkflowMain.groovy to provide better `--help`
+
+## [v0.1.0]
+### Changed
+- `sample_name` to `sample_id` throughout to mathc MinKNOW samplesheet.
+### Added
+- Singularity profile include in base config.
+- Numerous other changes that have been lost to the mists of time.
+
+## [v0.0.7]
+### Added
+- Fastqingress module for common handling of (possibly
+  multiplexed) inputs.
+- Optimized container size through removal of various
+  conda cruft.
+### Changed
+- Use mamba by default for building conda environments.
+- Cut down README to items specific to workflow.
+### Fixed
+- Incorrect specification of conda environment file in Nextflow config.
+
+## [v0.0.6]
+### Changed
+- Explicitely install into base conda env
+
+## [v0.0.5]
+### Added
+- Software versioning report example.
+
+## [v0.0.4]
+### Changed
+- Version bump to test CI.
+
+## [v0.0.3]
+### Changed
+- Moved all CI to templates.
+- Use canned aplanat report components.
+
+## [v0.0.2]
+### Added
+- CI release checks.
+- Create pre-releases in CI from dev branch.
+
+## [v0.0.1]
+
+First release.