Merge branch 'tu-ruo-next-sample-sheet' into 'dev'

Template updates (sample_sheet)

See merge request epi2melabs/workflows/wf-transcriptomes!278

bin/workflow_glue/__init__.py

@@ -77,7 +77,7 @@ def cli():
     # importing everything can take time, try to shortcut
     if len(sys.argv) > 1:
         components = get_components(allowed_components=[sys.argv[1]])
-        if not sys.argv[1] in components:
+        if sys.argv[1] not in components:
             logger.warn("Importing all modules, this may take some time.")
             components = get_components()
     else:

bin/workflow_glue/wfg_helpers/check_sample_sheet.py

@@ -1,11 +1,19 @@
-"""Check if a sample sheet is valid."""
+"""Check if a sample sheet is valid.
+
+Loads validator classes from modules in the validators package to check various
+aspects of a sample sheet. If any validator fails, the sample sheet is invalid.
+"""
 import codecs
 import csv
+from importlib import import_module
+from inspect import getmembers, isclass
+import json
 import os
-import re
+from pkgutil import iter_modules
 import sys
 
 from ..util import get_named_logger, wf_parser  # noqa: ABS101
+from . import validators  # noqa: ABS101
 
 
 # Some Excel users save their CSV as UTF-8 (and occasionally for a reason beyond my
@@ -31,18 +39,51 @@ def determine_codec(f):
         return None  # will cause file to be opened with default encoding
 
 
-def main(args):
-    """Run the entry point."""
-    logger = get_named_logger("checkSheet")
+def load_validators(modules, wf_params):
+    """Load validator classes."""
+    validator_classes = []
+    for mod in modules:
+        for _, validator_class in getmembers(mod, _is_sample_sheet_validator):
+            validator_classes.append(validator_class(wf_params))
+    return validator_classes
 
-    barcodes = []
-    aliases = []
-    sample_types = []
-    analysis_groups = []
-    allowed_sample_types = [
-        "test_sample", "positive_control", "negative_control", "no_template_control"
+
+def load_validator_modules():
+    """Load all modules from the validators package."""
+    return [
+        import_module(f"{validators.__name__}.{module_info.name}")
+        for module_info in iter_modules(validators.__path__)
     ]
 
+
+def _is_sample_sheet_validator(obj):
+    """Return whether an object is a concrete sample sheet validator class.
+
+    `issubclass(cls, Base)` is true when `cls is Base`, so the base validator
+    class must be excluded explicitly.
+    """
+    return (
+        isclass(obj) and issubclass(obj, validators.SampleSheetValidator)
+        and obj is not validators.SampleSheetValidator
+    )
+
+
+def main(args):
+    """Run the sample sheet checks."""
+    logger = get_named_logger("checkSheet")
+    with open(args.wf_params_path) as f:
+        wf_params = json.load(f)
+    if args.required_sample_types:
+        wf_params['required_sample_types'] = args.required_sample_types
+
+    # `no_barcode` is currently passed as a CLI option,
+    # not through the workflow params JSON.
+    wf_params['no_barcode'] = args.no_barcode
+
+    validator_classes = load_validators(load_validator_modules(), wf_params)
+
+    rows = []
+
     if not os.path.exists(args.sample_sheet) or not os.path.isfile(args.sample_sheet):
         sys.stdout.write("Could not open sample sheet file.")
         sys.exit()
@@ -66,110 +107,25 @@ def main(args):
                 sys.exit()
             csv_reader = csv.DictReader(f)
             columns = csv_reader.fieldnames
-            alias_field = "alias"
-            required_fields = ["barcode"]
-            prohibited_fields = []
+            rows = list(csv_reader)
 
-            if args.no_barcode:
-                alias_field = "sample_name"
-                required_fields = []
-                prohibited_fields = ["alias", "barcode"]
-
-            required_fields.append(alias_field)
-
-            for field in prohibited_fields:
-                if field in columns:
-                    sys.stdout.write(
-                        f"'{field}' column must not be present with --no_barcode"
-                    )
-                    sys.exit()
-
-            for field in required_fields:
-                if field not in columns:
-                    sys.stdout.write(f"'{field}' column missing")
-                    sys.exit()
-
-            # Skip header row for n_row
-            for n_row, row in enumerate(csv_reader, start=1):
-                if len(row) != len(columns):
-                    sys.stdout.write(
-                        f"Unexpected number of cells in row number {n_row}"
-                    )
-                    sys.exit()
-                if not args.no_barcode:
-                    barcodes.append(row.get("barcode"))
-                aliases.append(row.get(alias_field))
-
-                # Optional fields check for not None, empty strings are falsey
-                sample_type = row.get("type")
-                if sample_type is not None:
-                    sample_types.append(sample_type)
-                analysis_group = row.get("analysis_group")
-                if analysis_group is not None:
-                    analysis_groups.append(analysis_group)
     except Exception as e:
         sys.stdout.write(f"Parsing error: {e}")
         sys.exit()
 
-    # check barcodes are correct format
-    for barcode in barcodes:
-        if not re.match(r'^barcode\d\d+$', barcode):
-            sys.stdout.write("values in 'barcode' column are incorrect format")
-            sys.exit()
+    # Run all the validators.
+    for v in validator_classes:
+        v.on_header(columns)
+    # Skip header row
+    for lineno, row in enumerate(rows, start=1):
+        for v in validator_classes:
+            v.add_sheet_row(row, lineno)
 
-    # check aliases are correct format
-    # for now we have decided they may not start with "barcode"
-    for alias in aliases:
-        if alias.startswith("barcode"):
-            sys.stdout.write(
-                f"values in '{alias_field}' column must "
-                "not begin with 'barcode'")
-            sys.exit()
-
-    # check barcodes are all the same length
-    if barcodes:
-        first_length = len(barcodes[0])
-        for barcode in barcodes[1:]:
-            if len(barcode) != first_length:
-                sys.stdout.write("values in 'barcode' column are different lengths")
-                sys.exit()
-
-    # check barcode and alias values are unique
-    if len(barcodes) > len(set(barcodes)):
-        sys.stdout.write("values in 'barcode' column not unique")
+    if not all((v.is_valid for v in validator_classes)):
+        for v in validator_classes:
+            for e in v.errors:
+                sys.stdout.write(f"{e}\n")
         sys.exit()
-    if len(aliases) > len(set(aliases)):
-        sys.stdout.write(f"values in '{alias_field}' column not unique")
-        sys.exit()
-
-    if sample_types:
-        # check if "type" column has unexpected values
-        unexp_type_vals = set(sample_types) - set(allowed_sample_types)
-
-        if unexp_type_vals:
-            sys.stdout.write(
-                f"found unexpected values in 'type' column: {unexp_type_vals}. "
-                f"Allowed values are: {allowed_sample_types}"
-            )
-            sys.exit()
-
-        if args.required_sample_types:
-            for required_type in args.required_sample_types:
-                if required_type not in allowed_sample_types:
-                    sys.stdout.write(f"Not an allowed sample type: {required_type}")
-                    sys.exit()
-                if sample_types.count(required_type) < 1:
-                    sys.stdout.write(
-                        f"Sample sheet requires at least 1 of {required_type}")
-                    sys.exit()
-    if analysis_groups:
-        # if there was a "analysis_group" column, make sure it had values for all
-        # samples
-        if not all(analysis_groups):
-            sys.stdout.write(
-                "if an 'analysis_group' column exists, it needs values in each row"
-            )
-            sys.exit()
 
     logger.info(f"Checked sample sheet {args.sample_sheet}.")
 
@@ -177,7 +133,8 @@ def main(args):
 def argparser():
     """Argument parser for entrypoint."""
     parser = wf_parser("check_sample_sheet")
-    parser.add_argument("sample_sheet", help="Sample sheet to check")
+    parser.add_argument("sample_sheet", help="Sample sheet path to check")
+    parser.add_argument("wf_params_path", help="Path to WF params JSON")
     parser.add_argument(
         "--required_sample_types",
         help="List of required sample types. Each sample type provided must "

bin/workflow_glue/wfg_helpers/validators/__init__.py

@@ -0,0 +1,10 @@
+"""Sample sheet validators.
+
+Add workflow-specific validators by adding modules to this package. Each module
+can define one or more concrete subclasses of `SampleSheetValidator`; these will be
+discovered automatically by `check_sample_sheet`.
+"""
+
+from .default import SampleSheetValidator  # noqa: ABS101
+
+__all__ = ["SampleSheetValidator"]

bin/workflow_glue/wfg_helpers/validators/default.py

@@ -0,0 +1,251 @@
+"""Required sample sheet validators.
+
+These should be applied to sample sheets from all workflows.
+"""
+
+from abc import ABC
+import re
+
+
+# Define the base class for sample sheet validators
+class SampleSheetValidator(ABC):
+    """Base class for sample sheet validators."""
+
+    def __init__(self, wf_params):
+        """Initialize the validator with workflow parameters."""
+        self.wf_params = wf_params
+        self.errors = []
+
+    def log_error(self, msg, column=None, lineno=None):
+        """Log an error message with optional column and line context."""
+        context = []
+        if column is not None:
+            context.append(f"column: {column}")
+        if lineno is not None:
+            context.append(f"line: {lineno}")
+        if context:
+            msg = f"{msg} ({', '.join(context)})"
+        self.errors.append(msg)
+
+    def on_header(self, header):
+        """Handle header line."""
+        pass
+
+    def add_sheet_row(self, row, lineno):
+        """Handle a single row from the sample sheet."""
+        pass
+
+    @property
+    def is_valid(self):
+        """Check if the sample sheet is valid according to this validator."""
+        if self.errors:
+            return False
+        return True
+
+
+class ProhibitedColumns(SampleSheetValidator):
+    """Check for prohibited columns."""
+
+    def __init__(self, wf_params):
+        """Initialize with workflow parameters."""
+        super().__init__(wf_params)
+
+    def on_header(self, header):
+        """Don't allow `barcode` and `alias` when `--no_barcode` set."""
+        if self.wf_params.get("no_barcode"):
+            for field in ["alias", "barcode"]:
+                if field in header:
+                    self.log_error(
+                        "Column must not be present with --no_barcode",
+                        column=field,
+                    )
+
+
+class RequiredColumns(SampleSheetValidator):
+    """Check for required columns."""
+
+    def on_header(self, header):
+        """Check for required columns."""
+        self.header = header
+
+        required = ["barcode", "alias"]
+        if self.wf_params.get("no_barcode"):
+            required = ['sample_name']
+
+        for req in required:
+            if req not in header:
+                self.log_error("Column missing", column=req)
+
+    def add_sheet_row(self, row, lineno):
+        """Check for consistent number of columns."""
+        if len(row) != len(self.header):
+            self.log_error("Unexpected number of cells in row", lineno=lineno)
+
+
+class SampleTypeRules(SampleSheetValidator):
+    """Check sample types."""
+
+    ALLOWED = {
+        "test_sample", "positive_control", "negative_control", "no_template_control"
+    }
+
+    def __init__(self, wf_params):
+        """Initialize with workflow parameters."""
+        super().__init__(wf_params)
+        self.types = []
+        self.unexpected_types = []
+        self.required = wf_params.get("required_sample_types", [])
+
+    def add_sheet_row(self, row, lineno):
+        """Collect sample types."""
+        t = row.get("type")
+        if t:
+            self.types.append(t)
+            if t not in self.ALLOWED:
+                self.unexpected_types.append((t, lineno))
+
+    @property
+    def is_valid(self):
+        """Check if the sample types are valid."""
+        valid = True
+        if self.unexpected_types:
+            for t, lineno in self.unexpected_types:
+                self.log_error(
+                    f"Unexpected value: {t}", column="type", lineno=lineno)
+            valid = False
+        for required in self.required:
+            if required not in self.ALLOWED:
+                self.log_error(f"Not an allowed sample type: {required}")
+                valid = False
+            elif required not in self.types:
+                self.log_error(f"Sample sheet requires at least 1 of '{required}'")
+                valid = False
+        return valid
+
+
+class NonEmptySampleSheet(SampleSheetValidator):
+    """Check that the sample sheet has at least one data row."""
+
+    def __init__(self, wf_params):
+        """Initialize with workflow parameters."""
+        super().__init__(wf_params)
+        self.row_count = 0
+
+    def add_sheet_row(self, row, lineno):
+        """Count sample rows."""
+        self.row_count += 1
+
+    @property
+    def is_valid(self):
+        """Check the sample sheet has at least one data row."""
+        if self.row_count == 0 and not self.errors:
+            self.log_error("Sample sheet must contain at least one data row")
+        return super().is_valid
+
+
+class BarcodeRules(SampleSheetValidator):
+    """Check barcode rules."""
+
+    def __init__(self, wf_params):
+        """Initialize with workflow parameters."""
+        super().__init__(wf_params)
+        self.first_len = None
+        self.barcodes = []
+
+    def add_sheet_row(self, row, lineno):
+        """Check barcode rules."""
+        if self.wf_params.get("no_barcode"):
+            return
+
+        bc = row.get("barcode")
+        if bc is None:
+            self.log_error("Column missing", column="barcode")
+            return
+        if bc in self.barcodes:
+            self.log_error(
+                f"Value not unique: {bc}", column="barcode", lineno=lineno
+            )
+        self.barcodes.append(bc)
+        if not re.match(r"^barcode\d\d+$", bc):
+            self.log_error(
+                f"Value has incorrect format: {bc}",
+                column="barcode",
+                lineno=lineno,
+            )
+        if self.first_len is None:
+            self.first_len = len(bc)
+        elif len(bc) != self.first_len:
+            self.log_error(
+                f"Values are different lengths: {bc}",
+                column="barcode",
+                lineno=lineno,
+            )
+
+
+class AliasRules(SampleSheetValidator):
+    """Alias rules."""
+
+    ALIAS_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$")
+
+    def __init__(self, wf_params):
+        """Initialize with workflow parameters."""
+        super().__init__(wf_params)
+        self.aliases = set()
+
+    def add_sheet_row(self, row, lineno):
+        """Check alias rules."""
+        if self.wf_params.get("no_barcode"):
+            return
+
+        alias = row.get("alias")
+        if alias is None:
+            self.log_error("Column missing", column="alias")
+            return
+        if alias in self.aliases:
+            self.log_error(
+                f"Value not unique: {alias}", column="alias", lineno=lineno
+            )
+        self.aliases.add(alias)
+        if not alias:
+            self.log_error(
+                f"Empty value: {alias}. Allowed values start with letters or numbers "
+                "and may contain only letters, numbers, '.', '_' or '-'",
+                column="alias",
+                lineno=lineno,
+            )
+        if not self.ALIAS_PATTERN.match(alias):
+            self.log_error(
+                f"Invalid value {alias}. Allowed values start with letters or numbers "
+                "and may contain only letters, numbers, '.', '_' or '-'",
+                column="alias",
+                lineno=lineno,
+            )
+        if alias.startswith("barcode"):
+            self.log_error(
+                f"Value must not begin with 'barcode': {alias}",
+                column="alias",
+                lineno=lineno,
+            )
+
+
+class AnalysisGroupCompleteness(SampleSheetValidator):
+    """Analysis groups."""
+
+    def __init__(self, wf_params):
+        """Initialize with workflow parameters."""
+        super().__init__(wf_params)
+        self.has_analysis_groups = False
+
+    def on_header(self, header):
+        """Check for required columns."""
+        if "analysis_group" in header:
+            self.has_analysis_groups = True
+
+    def add_sheet_row(self, row, lineno):
+        """Check analysis group completeness."""
+        if self.has_analysis_groups and not row.get("analysis_group"):
+            self.log_error(
+                "Column exists but needs values in each row",
+                column="analysis_group",
+                lineno=lineno,
+            )

lib/ingress.nf

@@ -1,4 +1,5 @@
 import java.nio.file.NoSuchFileException
+import groovy.json.JsonBuilder
 
 import ArgumentParser
 
@@ -1341,7 +1342,7 @@ def get_sample_sheet(Path sample_sheet, ArrayList required_sample_types) {
  * @return: string (optional)
  */
 process validate_sample_sheet {
-    publishDir params.out_dir, mode: 'copy', overwrite: true
+    publishDir params.out_dir, pattern: 'sample_sheet.csv', mode: 'copy', overwrite: true
     cpus 1
     label "ingress"
     label "wf_common"
@@ -1355,8 +1356,10 @@ process validate_sample_sheet {
     script:
     String req_types_arg = required_sample_types ? "--required_sample_types "+required_sample_types.join(" ") : ""
     String no_barcode_arg = no_barcode ? "--no_barcode" : ""
+    def paramsJSON = new JsonBuilder(params).toPrettyString().replaceAll("'", "'\\\\''")
     """
-    workflow-glue check_sample_sheet sample_sheet.csv $req_types_arg $no_barcode_arg
+    echo '$paramsJSON' > params.json
+    workflow-glue check_sample_sheet sample_sheet.csv params.json $req_types_arg $no_barcode_arg
     """
 }