Merge branch 'template-updates' into 'dev'

template updates

See merge request epi2melabs/workflows/wf-transcriptomes!113

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [unreleased]
+### Changed
+- Any sample aliases that contain spaces will be replaced with underscores.
+
 ## [v0.2.0]
 ### Changed
 - GitHub issue templates

bin/workflow_glue/__init__.py

@@ -4,7 +4,7 @@ import glob
 import importlib
 import os
 
-from workflow_glue.util import _log_level, get_main_logger  # noqa: ABS101
+from .util import _log_level, get_main_logger  # noqa: ABS101
 
 
 __version__ = "0.0.1"
@@ -13,14 +13,24 @@ _package_name = "workflow_glue"
 
 def get_components():
     """Find a list of workflow command scripts."""
+    logger = get_main_logger(_package_name)
     path = os.path.dirname(os.path.abspath(__file__))
     components = list()
     for fname in glob.glob(os.path.join(path, "*.py")):
         name = os.path.splitext(os.path.basename(fname))[0]
         if name in ("__init__", "util"):
             continue
-        mod = importlib.import_module(f"{_package_name}.{name}")
+
-        # if there's a main() and and argparser() that's good enough for us.
+        # leniently attempt to import module
+        try:
+            mod = importlib.import_module(f"{_package_name}.{name}")
+        except ModuleNotFoundError as e:
+            # if imports cannot be satisifed, refuse to add the component
+            # rather than exploding
+            logger.warn(f"Could not load {name} due to missing module {e.name}")
+            continue
+
+        # if theres a main() and and argparser() thats good enough for us.
         try:
             req = "main", "argparser"
             if all(callable(getattr(mod, x)) for x in req):

bin/workflow_glue/check_sample_sheet.py

@@ -1,10 +1,35 @@
 """Check if a sample sheet is valid."""
+import codecs
 import csv
+import os
 import sys
 
 from .util import get_named_logger, wf_parser  # noqa: ABS101
 
 
+# Some Excel users save their CSV as UTF-8 (and occasionally for a reason beyond my
+# comprehension, UTF-16); Excel then adds a byte order mark (unnecessarily for UTF-8
+# I should add). If we do not handle this with the correct encoding, the mark will
+# appear in the parsed data, causing the header to be malformed.
+# See CW-2310
+def determine_codec(f):
+    """Peek at a file and return an appropriate reading codec."""
+    with open(f, 'rb') as f_bytes:
+        # Could use chardet here if we need to expand codec support
+        initial_bytes = f_bytes.read(8)
+
+        for codec, encoding_name in [
+            [codecs.BOM_UTF8, "utf-8-sig"],  # use the -sig codec to drop the mark
+            [codecs.BOM_UTF16_BE, "utf-16"],  # don't specify LE or BE to drop mark
+            [codecs.BOM_UTF16_LE, "utf-16"],
+            [codecs.BOM_UTF32_BE, "utf-32"],  # handle 32 for completeness
+            [codecs.BOM_UTF32_LE, "utf-32"],  # again skip LE or BE to drop mark
+        ]:
+            if initial_bytes.startswith(codec):
+                return encoding_name
+        return None  # will cause file to be opened with default encoding
+
+
 def main(args):
     """Run the entry point."""
     logger = get_named_logger("checkSheet")
@@ -14,10 +39,15 @@ def main(args):
     sample_types = []
     allowed_sample_types = [
         "test_sample", "positive_control", "negative_control", "no_template_control"
-        ]
+    ]
+
+    if not os.path.exists(args.sample_sheet) or not os.path.isfile(args.sample_sheet):
+        sys.stdout.write(f"Could not open sample sheet '{args.sample_sheet}'.")
+        sys.exit()
 
     try:
-        with open(args.sample_sheet, "r") as f:
+        encoding = determine_codec(args.sample_sheet)
+        with open(args.sample_sheet, "r", encoding=encoding) as f:
             csv_reader = csv.DictReader(f)
             n_row = 0
             for row in csv_reader:

lib/fastqingress.nf

@@ -50,7 +50,17 @@ def fastq_ingress(Map arguments)
     def ch_result
     if (margs.fastcat_stats) {
         // run fastcat regardless of input type
-        ch_result = fastcat(ch_input.reads_found, margs["fastcat_extra_args"])
+        ch_result = fastcat(ch_input.reads_found, margs["fastcat_extra_args"]).map {
+            meta, reads, stats ->
+                // extract run_ids parsed by fastcat into metadata
+                ArrayList run_ids = stats.resolve("run_ids").splitText().collect {
+                    it.strip()
+                }
+                // `meta + [...]` returns a new map which is handy to avoid any
+                // modifying-maps-in-closures weirdness
+                // See https://github.com/nextflow-io/nextflow/issues/2660
+                [meta + [run_ids: run_ids], reads, stats]
+        }
     } else {
         // the fastcat stats were not requested --> run fastcat only on directories with
         // more than one FASTQ file (and not on single files or directories with a
@@ -187,9 +197,11 @@ def watch_path(Map margs) {
 
 
 process move_or_compress {
-    label params.process_label
+    label "fastq_ingress"
+    label "wf_common"
     cpus 1
     input:
+        // don't stage `input` with a literal because we check the file extension
         tuple val(meta), path(input)
     output:
         tuple val(meta), path("seqs.fastq.gz")
@@ -199,21 +211,22 @@ process move_or_compress {
             // we need to take into account that the file could already be named
             // "seqs.fastq.gz" in which case `mv` would fail
             """
-            [ "$input" == "$out" ] || mv $input $out
+            [ "$input" == "$out" ] || mv "$input" $out
             """
         } else {
             """
-            cat $input | bgzip -@ $task.cpus > $out
+            cat "$input" | bgzip -@ $task.cpus > $out
             """
         }
 }
 
 
 process fastcat {
-    label params.process_label
+    label "fastq_ingress"
+    label "wf_common"
     cpus 3
     input:
-        tuple val(meta), path(input)
+        tuple val(meta), path("input")
         val extra_args
     output:
         tuple val(meta), path("seqs.fastq.gz"), path("fastcat_stats")
@@ -227,8 +240,9 @@ process fastcat {
             -r $fastcat_stats_outdir/per-read-stats.tsv \
             -f $fastcat_stats_outdir/per-file-stats.tsv \
             $extra_args \
-            $input \
+            input \
             | bgzip -@ $task.cpus > $out
+        csvtk cut -tf runid $fastcat_stats_outdir/per-read-stats.tsv | csvtk del-header | sort | uniq > $fastcat_stats_outdir/run_ids
         """
 }
 
@@ -373,10 +387,13 @@ Map create_metamap(Map arguments) {
         kwargs: [
             "barcode": null,
             "type": "test_sample",
+            "run_ids": [],
         ],
         name: "create_metamap",
     )
-    return parser.parse_known_args(arguments)
+    def metamap = parser.parse_known_args(arguments)
+    metamap['alias'] = metamap['alias'].replaceAll(" ","_")
+    return metamap
 }
 
 
@@ -430,15 +447,16 @@ def get_sample_sheet(Path sample_sheet, ArrayList required_sample_types) {
  * @return: string (optional)
  */
 process validate_sample_sheet {
-    label params.process_label
+    label "fastq_ingress"
+    label "wf_common"
     input:
-        path csv
+        path "sample_sheet.csv"
         val required_sample_types
     output: stdout
     script:
     String req_types_arg = required_sample_types ? "--required_sample_types "+required_sample_types.join(" ") : ""
     """
-    workflow-glue check_sample_sheet $csv $req_types_arg
+    workflow-glue check_sample_sheet sample_sheet.csv $req_types_arg
     """
 }
 

nextflow.config

@@ -26,14 +26,13 @@ params {
     sample_sheet = null
     aws_image_prefix = null
     aws_queue = null
-    process_label = "isoforms"
     analyse_unclassified = false
     version = false
 
     monochrome_logs = false
     validate_params = true
     show_hidden_params = false
-    schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf,process_label'
+    schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf'
 
     // Process cDNA reads using pychopper, turn off for direct RNA:
     direct_rna = false
@@ -105,6 +104,7 @@ params {
       ]
     agent = null
     container_sha = "sha203915eb4b4dd444cb2e845d0b9f7814e26b7b5c"
+    common_sha = "sha0fa3896acb70eecc0d432c91a1516d596a87741c"
     }
 }
 
@@ -135,6 +135,10 @@ process {
     withLabel:isoforms {
         container = "ontresearch/wf-transcriptomes:${params.wf.container_sha}"
     }
+    withLabel:wf_common {
+        container = "ontresearch/wf-common:${params.wf.common_sha}"
+    }
+
     shell = ['/bin/bash', '-euo', 'pipefail']
 }
 
@@ -174,6 +178,9 @@ profiles {
             withLabel:isoforms {
                 container = "${params.aws_image_prefix}-wf-transcriptomes:${params.wf.container_sha}-root"
             }
+            withLabel:wf_common {
+                container = "${params.aws_image_prefix}-wf-common:${params.wf.common_sha}-root"
+            } 
             shell = ['/bin/bash', '-euo', 'pipefail']
         }
     }