Merge branch 'pre-release-ref-only' into 'dev'

pre-release ref-only

See merge request epi2melabs/workflows/wf-amplicon!2

.gitlab-ci.yml

@@ -1,173 +1,37 @@
 # Include shared CI
 include:
-    - project: "epi2melabs/ci-templates"
-      file: "wf-containers.yaml"
+  - project: "epi2melabs/ci-templates"
+    file: "wf-containers.yaml"
 
 variables:
-    # We'll use the single-file case for these runs
-    NF_WORKFLOW_OPTS: "--fastq test_data/reads.fastq.gz"
-    CI_FLAVOUR: "new" # set to "classic" for old-style CI
-    TEST_CMD: "bash test/run_fastq_ingress_test.sh"
-    S3_TEST_DATA: "s3://ont-exd-int-s3-euwst1-epi2me-labs/wf-template/cw-1019/test_data"
-
-
-# Remove this block in downstream templates
-singularity-run:
-    tags: [] # no need for big ram
-# end
-
+  NF_WORKFLOW_OPTS:
+    "--fastq test_data/fastq --reference test_data/reference.fasta \
+    --threads 2"
+  CI_FLAVOUR: "new"
 
 docker-run:
-
-    # Remove this directive in downstream templates
-    tags: [] # no need for big ram
-
-    # Define a 1D job matrix to inject a variable named MATRIX_NAME into
-    #   the CI environment, we can use the value of MATRIX_NAME to determine
-    #   which options to apply as part of the rules block below
-    # NOTE There is a slightly cleaner way to define this matrix to include
-    #   the variables, but it is broken when using long strings! See CW-756
-    parallel:
-        matrix:
-            # TODO: do we really need to run all from s3?
-            - MATRIX_NAME: [
-                "single-file", "single-file-s3",
-                "case01", "case01-s3",
-                "case02", "case02-s3",
-                "case03", "case03-s3",
-                "case04", "case04-s3",
-                "case05", "case05-s3",
-                "case06", "case06-s3",
-                "case07", "case07-s3",
-                "case08", "case08-s3", case08-unclassified, case08-unclassified-s3,
-                "case09", "case09-s3",
-                "case10", "case10-s3",
-                "case11", "case11-s3",
-                ]
-    rules:
-        # NOTE As we're overriding the rules block for the included docker-run
-        #   we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run
-        #   being incorrectly scheduled for "detached merge request pipelines" etc.
-        - if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template")
-          when: never
-        - if: $MATRIX_NAME == "single-file"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/reads.fastq.gz"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/reads.fastq.gz $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "single-file-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/reads.fastq.gz"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/reads.fastq.gz $$PWD/$$CI_PROJECT_NAME
-
-        - if: $MATRIX_NAME == "case01"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case01"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case01 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case01-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case01"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case01 $$PWD/$$CI_PROJECT_NAME
-
-        - if: $MATRIX_NAME == "case02"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case02"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case02 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case02-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case02"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case02 $$PWD/$$CI_PROJECT_NAME
-
-        - if: $MATRIX_NAME == "case03"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case03"
-              ASSERT_NEXTFLOW_FAILURE: "1"
-              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain FASTQ files and sub-directories."
-        - if: $MATRIX_NAME == "case03-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case03"
-              ASSERT_NEXTFLOW_FAILURE: "1"
-              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain FASTQ files and sub-directories."
-
-        - if: $MATRIX_NAME == "case04"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case04"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case04 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case04-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case04"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case04 $$PWD/$$CI_PROJECT_NAME
-
-        - if: $MATRIX_NAME == "case05"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case05 --sample_sheet test_data/fastq_ingress/case05/sample_sheet.csv"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case05 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case05-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case05 --sample_sheet $S3_TEST_DATA/fastq_ingress/case05/sample_sheet.csv"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case05 $$PWD/$$CI_PROJECT_NAME $S3_TEST_DATA/fastq_ingress/case05/sample_sheet.csv
-
-        - if: $MATRIX_NAME == "case06"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case06 --sample_sheet test_data/fastq_ingress/case06/sample_sheet.csv"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case06 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case06-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case06 --sample_sheet $S3_TEST_DATA/fastq_ingress/case06/sample_sheet.csv"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case06 $$PWD/$$CI_PROJECT_NAME $S3_TEST_DATA/fastq_ingress/case06/sample_sheet.csv
-
-        - if: $MATRIX_NAME == "case07"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case07 --sample_sheet test_data/fastq_ingress/case07/sample_sheet.csv"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case07 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case07-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case07 --sample_sheet $S3_TEST_DATA/fastq_ingress/case07/sample_sheet.csv"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case07 $$PWD/$$CI_PROJECT_NAME $S3_TEST_DATA/fastq_ingress/case07/sample_sheet.csv
-
-        - if: $MATRIX_NAME == "case08"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case08"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case08-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case08"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case08-unclassified"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case08 --analyse_unclassified"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case08-unclassified-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case08 --analyse_unclassified"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case08 $$PWD/$$CI_PROJECT_NAME
-
-        - if: $MATRIX_NAME == "case09"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case09"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD test_data/fastq_ingress/case09 $$PWD/$$CI_PROJECT_NAME
-        - if: $MATRIX_NAME == "case09-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case09"
-              AFTER_NEXTFLOW_CMD: $TEST_CMD $S3_TEST_DATA/fastq_ingress/case09 $$PWD/$$CI_PROJECT_NAME
-
-        - if: $MATRIX_NAME == "case10"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case10"
-              ASSERT_NEXTFLOW_FAILURE: "1"
-              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain more than one level of sub-directories with FASTQ files."
-        - if: $MATRIX_NAME == "case10-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case10"
-              ASSERT_NEXTFLOW_FAILURE: "1"
-              ASSERT_NEXTFLOW_FAILURE_REXP: "Input directory '.*' cannot contain more than one level of sub-directories with FASTQ files."
-
-        - if: $MATRIX_NAME == "case11"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq test_data/fastq_ingress/case11 --sample_sheet test_data/fastq_ingress/case11/sample_sheet.csv"
-              ASSERT_NEXTFLOW_FAILURE: "1"
-              ASSERT_NEXTFLOW_FAILURE_REXP: "Invalid sample sheet: values in 'alias' column not unique"
-        - if: $MATRIX_NAME == "case11-s3"
-          variables:
-              NF_WORKFLOW_OPTS: "--fastq $S3_TEST_DATA/fastq_ingress/case11 --sample_sheet $S3_TEST_DATA/fastq_ingress/case11/sample_sheet.csv"
-              ASSERT_NEXTFLOW_FAILURE: "1"
-              ASSERT_NEXTFLOW_FAILURE_REXP: "Invalid sample sheet: values in 'alias' column not unique"
+  # Remove this directive in downstream templates
+  tags: [large_ram] # no need for big ram
+
+  # Define a 1D job matrix to inject a variable named MATRIX_NAME into
+  #   the CI environment, we can use the value of MATRIX_NAME to determine
+  #   which options to apply as part of the rules block below
+  # NOTE There is a slightly cleaner way to define this matrix to include
+  #   the variables, but it is broken when using long strings! See CW-756
+  parallel:
+    matrix:
+      - MATRIX_NAME: ["ref"]
+  rules:
+    # NOTE As we're overriding the rules block for the included docker-run
+    #   we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run
+    #   being incorrectly scheduled for "detached merge request pipelines" etc.
+    - if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template")
+      when: never
+    - if: $MATRIX_NAME == "ref"
+      variables:
+        NF_WORKFLOW_OPTS:
+          "--fastq test_data/fastq --reference test_data/reference.fasta \
+          --threads 2 --reads_downsampling_size 2000"
+        NF_PROCESS_FILES: >
+          main.nf
+          modules/local/pipeline.nf

.pre-commit-config.yaml

@@ -18,9 +18,10 @@ repos:
         additional_dependencies:
           - epi2melabs
   - repo: https://github.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 5.0.4
     hooks:
       - id: flake8
+        pass_filenames: false
         additional_dependencies:
           - flake8-rst-docstrings
           - flake8-docstrings
@@ -31,4 +32,9 @@ repos:
           - flake8-builtins
           - flake8-absolute-import
           - flake8-print
-        entry: flake8 bin --import-order-style google --statistics --max-line-length 88
+        args: [
+            "bin",
+            "--import-order-style=google",
+            "--statistics",
+            "--max-line-length=88",
+        ]

CHANGELOG.md

@@ -4,6 +4,6 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [v0.0.1]
+## [v0.1.0]
 
 First release.