From a58d08b6c44495d75eea5b607c318e9a15ad8e7a Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Tue, 25 Jul 2023 08:33:01 +0000 Subject: [PATCH] template updates --- CHANGELOG.md | 4 +++ bin/workflow_glue/__init__.py | 16 ++++++++-- bin/workflow_glue/check_sample_sheet.py | 34 ++++++++++++++++++-- lib/fastqingress.nf | 42 ++++++++++++++++++------- nextflow.config | 11 +++++-- 5 files changed, 88 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e04b3f6..80469d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Changed +- Any sample aliases that contain spaces will be replaced with underscores. + ## [v0.2.0] ### Changed - GitHub issue templates diff --git a/bin/workflow_glue/__init__.py b/bin/workflow_glue/__init__.py index ffc7c80..30febca 100755 --- a/bin/workflow_glue/__init__.py +++ b/bin/workflow_glue/__init__.py @@ -4,7 +4,7 @@ import glob import importlib import os -from workflow_glue.util import _log_level, get_main_logger # noqa: ABS101 +from .util import _log_level, get_main_logger # noqa: ABS101 __version__ = "0.0.1" @@ -13,14 +13,24 @@ _package_name = "workflow_glue" def get_components(): """Find a list of workflow command scripts.""" + logger = get_main_logger(_package_name) path = os.path.dirname(os.path.abspath(__file__)) components = list() for fname in glob.glob(os.path.join(path, "*.py")): name = os.path.splitext(os.path.basename(fname))[0] if name in ("__init__", "util"): continue - mod = importlib.import_module(f"{_package_name}.{name}") - # if there's a main() and and argparser() that's good enough for us. + + # leniently attempt to import module + try: + mod = importlib.import_module(f"{_package_name}.{name}") + except ModuleNotFoundError as e: + # if imports cannot be satisifed, refuse to add the component + # rather than exploding + logger.warn(f"Could not load {name} due to missing module {e.name}") + continue + + # if theres a main() and and argparser() thats good enough for us. try: req = "main", "argparser" if all(callable(getattr(mod, x)) for x in req): diff --git a/bin/workflow_glue/check_sample_sheet.py b/bin/workflow_glue/check_sample_sheet.py index 2bd1779..e9d956f 100755 --- a/bin/workflow_glue/check_sample_sheet.py +++ b/bin/workflow_glue/check_sample_sheet.py @@ -1,10 +1,35 @@ """Check if a sample sheet is valid.""" +import codecs import csv +import os import sys from .util import get_named_logger, wf_parser # noqa: ABS101 +# Some Excel users save their CSV as UTF-8 (and occasionally for a reason beyond my +# comprehension, UTF-16); Excel then adds a byte order mark (unnecessarily for UTF-8 +# I should add). If we do not handle this with the correct encoding, the mark will +# appear in the parsed data, causing the header to be malformed. +# See CW-2310 +def determine_codec(f): + """Peek at a file and return an appropriate reading codec.""" + with open(f, 'rb') as f_bytes: + # Could use chardet here if we need to expand codec support + initial_bytes = f_bytes.read(8) + + for codec, encoding_name in [ + [codecs.BOM_UTF8, "utf-8-sig"], # use the -sig codec to drop the mark + [codecs.BOM_UTF16_BE, "utf-16"], # don't specify LE or BE to drop mark + [codecs.BOM_UTF16_LE, "utf-16"], + [codecs.BOM_UTF32_BE, "utf-32"], # handle 32 for completeness + [codecs.BOM_UTF32_LE, "utf-32"], # again skip LE or BE to drop mark + ]: + if initial_bytes.startswith(codec): + return encoding_name + return None # will cause file to be opened with default encoding + + def main(args): """Run the entry point.""" logger = get_named_logger("checkSheet") @@ -14,10 +39,15 @@ def main(args): sample_types = [] allowed_sample_types = [ "test_sample", "positive_control", "negative_control", "no_template_control" - ] + ] + + if not os.path.exists(args.sample_sheet) or not os.path.isfile(args.sample_sheet): + sys.stdout.write(f"Could not open sample sheet '{args.sample_sheet}'.") + sys.exit() try: - with open(args.sample_sheet, "r") as f: + encoding = determine_codec(args.sample_sheet) + with open(args.sample_sheet, "r", encoding=encoding) as f: csv_reader = csv.DictReader(f) n_row = 0 for row in csv_reader: diff --git a/lib/fastqingress.nf b/lib/fastqingress.nf index 4d5a94f..42e6f5c 100644 --- a/lib/fastqingress.nf +++ b/lib/fastqingress.nf @@ -50,7 +50,17 @@ def fastq_ingress(Map arguments) def ch_result if (margs.fastcat_stats) { // run fastcat regardless of input type - ch_result = fastcat(ch_input.reads_found, margs["fastcat_extra_args"]) + ch_result = fastcat(ch_input.reads_found, margs["fastcat_extra_args"]).map { + meta, reads, stats -> + // extract run_ids parsed by fastcat into metadata + ArrayList run_ids = stats.resolve("run_ids").splitText().collect { + it.strip() + } + // `meta + [...]` returns a new map which is handy to avoid any + // modifying-maps-in-closures weirdness + // See https://github.com/nextflow-io/nextflow/issues/2660 + [meta + [run_ids: run_ids], reads, stats] + } } else { // the fastcat stats were not requested --> run fastcat only on directories with // more than one FASTQ file (and not on single files or directories with a @@ -187,9 +197,11 @@ def watch_path(Map margs) { process move_or_compress { - label params.process_label + label "fastq_ingress" + label "wf_common" cpus 1 input: + // don't stage `input` with a literal because we check the file extension tuple val(meta), path(input) output: tuple val(meta), path("seqs.fastq.gz") @@ -199,21 +211,22 @@ process move_or_compress { // we need to take into account that the file could already be named // "seqs.fastq.gz" in which case `mv` would fail """ - [ "$input" == "$out" ] || mv $input $out + [ "$input" == "$out" ] || mv "$input" $out """ } else { """ - cat $input | bgzip -@ $task.cpus > $out + cat "$input" | bgzip -@ $task.cpus > $out """ } } process fastcat { - label params.process_label + label "fastq_ingress" + label "wf_common" cpus 3 input: - tuple val(meta), path(input) + tuple val(meta), path("input") val extra_args output: tuple val(meta), path("seqs.fastq.gz"), path("fastcat_stats") @@ -227,8 +240,9 @@ process fastcat { -r $fastcat_stats_outdir/per-read-stats.tsv \ -f $fastcat_stats_outdir/per-file-stats.tsv \ $extra_args \ - $input \ + input \ | bgzip -@ $task.cpus > $out + csvtk cut -tf runid $fastcat_stats_outdir/per-read-stats.tsv | csvtk del-header | sort | uniq > $fastcat_stats_outdir/run_ids """ } @@ -373,10 +387,13 @@ Map create_metamap(Map arguments) { kwargs: [ "barcode": null, "type": "test_sample", + "run_ids": [], ], name: "create_metamap", ) - return parser.parse_known_args(arguments) + def metamap = parser.parse_known_args(arguments) + metamap['alias'] = metamap['alias'].replaceAll(" ","_") + return metamap } @@ -430,15 +447,16 @@ def get_sample_sheet(Path sample_sheet, ArrayList required_sample_types) { * @return: string (optional) */ process validate_sample_sheet { - label params.process_label - input: - path csv + label "fastq_ingress" + label "wf_common" + input: + path "sample_sheet.csv" val required_sample_types output: stdout script: String req_types_arg = required_sample_types ? "--required_sample_types "+required_sample_types.join(" ") : "" """ - workflow-glue check_sample_sheet $csv $req_types_arg + workflow-glue check_sample_sheet sample_sheet.csv $req_types_arg """ } diff --git a/nextflow.config b/nextflow.config index c4943b0..0d309d2 100644 --- a/nextflow.config +++ b/nextflow.config @@ -26,14 +26,13 @@ params { sample_sheet = null aws_image_prefix = null aws_queue = null - process_label = "isoforms" analyse_unclassified = false version = false monochrome_logs = false validate_params = true show_hidden_params = false - schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf,process_label' + schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf' // Process cDNA reads using pychopper, turn off for direct RNA: direct_rna = false @@ -105,6 +104,7 @@ params { ] agent = null container_sha = "sha203915eb4b4dd444cb2e845d0b9f7814e26b7b5c" + common_sha = "sha0fa3896acb70eecc0d432c91a1516d596a87741c" } } @@ -135,6 +135,10 @@ process { withLabel:isoforms { container = "ontresearch/wf-transcriptomes:${params.wf.container_sha}" } + withLabel:wf_common { + container = "ontresearch/wf-common:${params.wf.common_sha}" + } + shell = ['/bin/bash', '-euo', 'pipefail'] } @@ -174,6 +178,9 @@ profiles { withLabel:isoforms { container = "${params.aws_image_prefix}-wf-transcriptomes:${params.wf.container_sha}-root" } + withLabel:wf_common { + container = "${params.aws_image_prefix}-wf-common:${params.wf.common_sha}-root" + } shell = ['/bin/bash', '-euo', 'pipefail'] } }