From b4bba1432bc72ef7fe63e026c899a1bae97bd1d3 Mon Sep 17 00:00:00 2001
From: Sarah Griffiths <sarah.griffiths@nanoporetech.com>
Date: Fri, 14 Mar 2025 11:04:18 +0000
Subject: [PATCH] Template v5.5.0 updates

---
 .gitignore                                    |   1 +
 .pre-commit-config.yaml                       |  10 +-
 CHANGELOG.md                                  |   1 +
 bin/workflow_glue/__init__.py                 |  19 +-
 bin/workflow_glue/models/__init__.py          |   1 +
 bin/workflow_glue/models/common.py            | 206 ++++++++++
 bin/workflow_glue/util.py                     |   5 +-
 bin/workflow_glue/wfg_helpers/__init__.py     |   1 +
 .../check_bam_headers_in_dir.py               |   2 +-
 .../{ => wfg_helpers}/check_sample_sheet.py   |   9 +-
 .../{ => wfg_helpers}/check_xam_index.py      |   2 +-
 .../{ => wfg_helpers}/configure_igv.py        |   2 +-
 .../{ => wfg_helpers}/get_max_depth_locus.py  |   2 +-
 .../{ => wfg_helpers}/reheader_samstream.py   |  40 +-
 lib/NfcoreSchema.groovy                       |   9 +-
 lib/common.nf                                 |   2 +-
 lib/ingress.nf                                | 352 +++++++++++-------
 nextflow.config                               |   3 +-
 nextflow_schema.json                          |   4 +
 19 files changed, 510 insertions(+), 161 deletions(-)
 create mode 100644 bin/workflow_glue/models/__init__.py
 create mode 100644 bin/workflow_glue/models/common.py
 create mode 100644 bin/workflow_glue/wfg_helpers/__init__.py
 rename bin/workflow_glue/{ => wfg_helpers}/check_bam_headers_in_dir.py (97%)
 rename bin/workflow_glue/{ => wfg_helpers}/check_sample_sheet.py (94%)
 rename bin/workflow_glue/{ => wfg_helpers}/check_xam_index.py (94%)
 rename bin/workflow_glue/{ => wfg_helpers}/configure_igv.py (99%)
 rename bin/workflow_glue/{ => wfg_helpers}/get_max_depth_locus.py (96%)
 rename bin/workflow_glue/{ => wfg_helpers}/reheader_samstream.py (85%)

diff --git a/.gitignore b/.gitignore
index be27172..6a66ee9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ template-workflow
 .*.swo
 *.pyc
 *.pyo
+.DS_store
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4626336..0d407af 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -9,14 +9,6 @@ repos:
         pass_filenames: false
         additional_dependencies:
           - epi2melabs==0.0.57
-      - id: build_models
-        name: build_models
-        entry: datamodel-codegen --strict-nullable --base-class workflow_glue.results_schema_helpers.BaseModel --use-subclass-enum --use-schema-description --disable-timestamp --input results_schema.yml --input-file-type openapi --output bin/workflow_glue/results_schema.py
-        language: python
-        files: 'results_schema.yml'
-        pass_filenames: false
-        additional_dependencies:
-          - datamodel-code-generator
   - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:
@@ -37,5 +29,5 @@ repos:
             "--import-order-style=google",
             "--statistics",
             "--max-line-length=88",
-            "--extend-exclude=bin/workflow_glue/results_schema.py",
+            "--per-file-ignores=bin/workflow_glue/models/*:NT001",
         ]
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee3940b..ae1e4ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased.
 - Updated recommended memory requirement.
 - Updated project description.
+- Reconciled workflow with wf-template v5.5.0.
 ### Fixed
 - `all_gene_counts.tsv` contained the DE counts results.
 
diff --git a/bin/workflow_glue/__init__.py b/bin/workflow_glue/__init__.py
index 3949151..5d7e7d3 100644
--- a/bin/workflow_glue/__init__.py
+++ b/bin/workflow_glue/__init__.py
@@ -2,6 +2,7 @@
 import argparse
 import glob
 import importlib
+import itertools
 import os
 import sys
 
@@ -11,13 +12,22 @@ from .util import _log_level, get_main_logger  # noqa: ABS101
 __version__ = "0.0.1"
 _package_name = "workflow_glue"
 
+HELPERS = "wfg_helpers"
+
 
 def get_components(allowed_components=None):
     """Find a list of workflow command scripts."""
     logger = get_main_logger(_package_name)
-    path = os.path.dirname(os.path.abspath(__file__))
+
+    # gather all python files in the current directory and the wfg_helpers
+    home_path = os.path.dirname(os.path.abspath(__file__))
+    standard_lib = os.path.join(home_path, HELPERS)
+    globs = itertools.chain.from_iterable((
+        glob.glob(os.path.join(path, "*.py"))
+        for path in (home_path, standard_lib)))
+
     components = dict()
-    for fname in glob.glob(os.path.join(path, "*.py")):
+    for fname in globs:
         name = os.path.splitext(os.path.basename(fname))[0]
         if name in ("__init__", "util"):
             continue
@@ -26,7 +36,10 @@ def get_components(allowed_components=None):
 
         # leniently attempt to import module
         try:
-            mod = importlib.import_module(f"{_package_name}.{name}")
+            if HELPERS in fname:
+                mod = importlib.import_module(f"{_package_name}.{HELPERS}.{name}")
+            else:
+                mod = importlib.import_module(f"{_package_name}.{name}")
         except ModuleNotFoundError as e:
             # if imports cannot be satisifed, refuse to add the component
             # rather than exploding
diff --git a/bin/workflow_glue/models/__init__.py b/bin/workflow_glue/models/__init__.py
new file mode 100644
index 0000000..dbbe078
--- /dev/null
+++ b/bin/workflow_glue/models/__init__.py
@@ -0,0 +1 @@
+"""A collection of scripts for results models."""
diff --git a/bin/workflow_glue/models/common.py b/bin/workflow_glue/models/common.py
new file mode 100644
index 0000000..04a217f
--- /dev/null
+++ b/bin/workflow_glue/models/common.py
@@ -0,0 +1,206 @@
+"""Common model classes used across all workflows."""
+from dataclasses import asdict, dataclass, field
+from decimal import Decimal
+from enum import Enum
+import json
+from typing import Any, Dict, List
+
+
+class SampleType(str, Enum):
+    """The type of the sample."""
+
+    no_template_control = "no_template_control"
+    positive_control = "positive_control"
+    negative_control = "negative_control"
+    test_sample = "test_sample"
+
+    def friendly_name(self):
+        """Convert sample type to string."""
+        return self.name.replace("_", " ").capitalize()
+
+
+@dataclass
+class SampleIdentifier:
+    """Additional identifiers for a sample."""
+
+    name: str = field(
+        metadata={
+            "title": "Identifier name",
+            "Description": "The name of the sample identifier"})
+    value: str = field(
+        metadata={
+            "title": "Identifier value",
+            "Description": "The value of the sample identifier"})
+
+
+@dataclass
+class CheckResult:
+    """
+    A result of some check the workflow has performed.
+
+    This can be at sample or workflow level.
+    """
+
+    check_category: str = field(
+        metadata={
+            "title": "Check category",
+            "description": "The category of the check"})
+    check_name: str = field(
+        metadata={
+            "title": "Check name",
+            "description": "The name of the check"})
+    check_pass: bool = field(
+        metadata={
+            "title": "Check pass",
+            "description": "If true the check has passed"})
+    check_threshold: str | None = field(
+        default=None, metadata={
+            "title": "Check threshold",
+            "description": "The threshold for the check, useful for reporting later"})
+
+    categories = {}
+
+    def friendly_check_category(self):
+        """Convert category to string."""
+        if self.check_category not in self.categories:
+            raise ValueError(f"{self.check_category} has no friendly name")
+        return self.categories[self.check_category]
+
+    def friendly_check_name(self):
+        """Convert check name to string."""
+        return self.check_name.replace("_", " ").capitalize()
+
+
+@dataclass
+class ResultsContents:
+    """Placeholder class for results contents."""
+
+    pass
+
+
+@dataclass
+class Sample:
+    """A sample sheet entry and its corresponding checks and related results."""
+
+    alias: str = field(
+        metadata={
+            "title": "Sample alias",
+            "description": "The alias for the sample given by the user"})
+    barcode: str = field(
+        metadata={
+            "title": "Sample barcode",
+            "description": "The physical barcode assigned to the sample"})
+    sample_type: SampleType = field(
+        metadata={
+            "title": "Sample type",
+            "description": "The type of the sample"})
+    sample_pass: bool = field(
+        metadata={
+            "title": "Sample pass",
+            "description": "If true the sample has passed workflow checks"})
+    additional_identifiers: List[SampleIdentifier] = field(
+        default_factory=list, metadata={
+            "title": "Additional sample identifiers",
+            "description": "Addition identifiers for the sample"})
+    sample_checks: list[CheckResult] = field(
+        default_factory=list, metadata={
+            "title": "Sample checks",
+            "description": "An array of checks performed on the sample"})
+    results: ResultsContents | None = field(
+        default=None, metadata={
+            "title": "Sample results",
+            "description": "Further specific workflow results for this sample"})
+    config:  Dict[str, Any] | None = field(
+        default=None, metadata={
+            "title": "Sample configuration",
+            "description": """Sample specific config parameters
+            used for running analysis"""})
+
+    def __post_init__(self):
+        """Determine overall status for a sample given the individual check results."""
+        self.sample_pass = all(
+            check.check_pass for check in self.sample_checks)
+
+    def get_sample_identifier(self, sample_identifier):
+        """Get a sample identifier given the identifier name."""
+        for indentifier in self.additional_identifiers:
+            if indentifier.name == sample_identifier:
+                return indentifier.value
+        raise KeyError("Sample identifier not found")
+
+    def set_sample_identifier(self, name, value):
+        """Set a sample identifier."""
+        sample_identifier = SampleIdentifier(
+            name=name,
+            value=value)
+        self.additional_identifiers.append(sample_identifier)
+        return self.additional_identifiers
+
+    def to_json(self, filename):
+        """Save class as JSON."""
+        with open(filename, 'w') as f:
+            json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder)
+
+
+@dataclass
+class RunStats:
+    """Basic run statistics for the entire run."""
+
+    total_reads: int | None = field(
+        default=None, metadata={
+            "title": "Total reads",
+            "description": "Total number of reads on run"})
+    total_ambiguous_reads: int | None = field(
+        default=None, metadata={
+            "title": "Total ambiguous reads",
+            "description": "Number of reads of unknown provenance"})
+    total_unaligned_reads: int | None = field(
+        default=None, metadata={
+            "title": "Total unaligned reads",
+            "description": "Number of unaligned reads"})
+
+
+@dataclass
+class WorkflowResult():
+    """
+    Definition for results that will be returned by this workflow.
+
+    This structure will be passed through by Gizmo speaking clients
+    as WorkflowInstance.results.
+    """
+
+    samples: list[Sample] = field(
+        metadata={
+            "title": "Samples",
+            "description": "Samples in this workflow instance"})
+    workflow_pass: bool | None = field(
+        default=None, metadata={
+            "title": "Workflow pass",
+            "description": "True if this workflow instance passes all checks"})
+    workflow_checks: list[CheckResult] = field(
+        default_factory=list, metadata={
+            "title": "Workflow checks",
+            "description": "An array of checks performed on the workflow instance"})
+    run_stats: RunStats | None = field(
+        default=None, metadata={
+            "title": "Samples",
+            "description": "Basic run statistics"})
+    client_fields: dict[str, Any] | None = field(
+        default_factory=dict, metadata={
+            "title": "Client fields",
+            "description": "Arbitrary key-value pairs provided by the client"})
+
+    def to_json(self, filename):
+        """Save class as JSON."""
+        with open(filename, 'w') as f:
+            json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder)
+
+
+class DecimalEncoder(json.JSONEncoder):
+    """This should probably be moved."""
+
+    def default(self, obj):
+        """Override the default method to handle Decimal objects."""
+        if isinstance(obj, Decimal):
+            return float(obj)
+        return super().default(obj)
diff --git a/bin/workflow_glue/util.py b/bin/workflow_glue/util.py
index 9359ac7..2e7d2da 100644
--- a/bin/workflow_glue/util.py
+++ b/bin/workflow_glue/util.py
@@ -1,8 +1,11 @@
-"""The odd helper function."""
+"""The odd helper function.
 
+Be careful what you place in here. This file is imported into all glue.
+"""
 import argparse
 import logging
 
+
 _log_name = None
 
 
diff --git a/bin/workflow_glue/wfg_helpers/__init__.py b/bin/workflow_glue/wfg_helpers/__init__.py
new file mode 100644
index 0000000..dd9367c
--- /dev/null
+++ b/bin/workflow_glue/wfg_helpers/__init__.py
@@ -0,0 +1 @@
+"""A collection of helper scripts common to workflows."""
diff --git a/bin/workflow_glue/check_bam_headers_in_dir.py b/bin/workflow_glue/wfg_helpers/check_bam_headers_in_dir.py
similarity index 97%
rename from bin/workflow_glue/check_bam_headers_in_dir.py
rename to bin/workflow_glue/wfg_helpers/check_bam_headers_in_dir.py
index 199e056..ff7f3b4 100644
--- a/bin/workflow_glue/check_bam_headers_in_dir.py
+++ b/bin/workflow_glue/wfg_helpers/check_bam_headers_in_dir.py
@@ -5,7 +5,7 @@ import sys
 
 import pysam
 
-from .util import get_named_logger, wf_parser  # noqa: ABS101
+from ..util import get_named_logger, wf_parser  # noqa: ABS101
 
 
 def main(args):
diff --git a/bin/workflow_glue/check_sample_sheet.py b/bin/workflow_glue/wfg_helpers/check_sample_sheet.py
similarity index 94%
rename from bin/workflow_glue/check_sample_sheet.py
rename to bin/workflow_glue/wfg_helpers/check_sample_sheet.py
index 8fec3f0..f75876f 100644
--- a/bin/workflow_glue/check_sample_sheet.py
+++ b/bin/workflow_glue/wfg_helpers/check_sample_sheet.py
@@ -5,7 +5,7 @@ import os
 import re
 import sys
 
-from .util import get_named_logger, wf_parser  # noqa: ABS101
+from ..util import get_named_logger, wf_parser  # noqa: ABS101
 
 
 # Some Excel users save their CSV as UTF-8 (and occasionally for a reason beyond my
@@ -106,6 +106,13 @@ def main(args):
             sys.stdout.write("values in 'barcode' column are incorrect format")
             sys.exit()
 
+    # check aliases are correct format
+    # for now we have decided they may not start with "barcode"
+    for alias in aliases:
+        if alias.startswith("barcode"):
+            sys.stdout.write("values in 'alias' column must not begin with 'barcode'")
+            sys.exit()
+
     # check barcodes are all the same length
     first_length = len(barcodes[0])
     for barcode in barcodes[1:]:
diff --git a/bin/workflow_glue/check_xam_index.py b/bin/workflow_glue/wfg_helpers/check_xam_index.py
similarity index 94%
rename from bin/workflow_glue/check_xam_index.py
rename to bin/workflow_glue/wfg_helpers/check_xam_index.py
index f9f631e..c6166de 100644
--- a/bin/workflow_glue/check_xam_index.py
+++ b/bin/workflow_glue/wfg_helpers/check_xam_index.py
@@ -5,7 +5,7 @@ import sys
 
 import pysam
 
-from .util import get_named_logger, wf_parser  # noqa: ABS101
+from ..util import get_named_logger, wf_parser  # noqa: ABS101
 
 
 def validate_xam_index(xam_file):
diff --git a/bin/workflow_glue/configure_igv.py b/bin/workflow_glue/wfg_helpers/configure_igv.py
similarity index 99%
rename from bin/workflow_glue/configure_igv.py
rename to bin/workflow_glue/wfg_helpers/configure_igv.py
index db735dd..a817e53 100644
--- a/bin/workflow_glue/configure_igv.py
+++ b/bin/workflow_glue/wfg_helpers/configure_igv.py
@@ -4,7 +4,7 @@ import json
 from pathlib import Path
 import sys
 
-from .util import get_named_logger, wf_parser  # noqa: ABS101
+from ..util import get_named_logger, wf_parser  # noqa: ABS101
 
 
 # Common variables
diff --git a/bin/workflow_glue/get_max_depth_locus.py b/bin/workflow_glue/wfg_helpers/get_max_depth_locus.py
similarity index 96%
rename from bin/workflow_glue/get_max_depth_locus.py
rename to bin/workflow_glue/wfg_helpers/get_max_depth_locus.py
index b873ea9..09e620b 100644
--- a/bin/workflow_glue/get_max_depth_locus.py
+++ b/bin/workflow_glue/wfg_helpers/get_max_depth_locus.py
@@ -5,7 +5,7 @@ import sys
 
 import pandas as pd
 
-from .util import get_named_logger, wf_parser  # noqa: ABS101
+from ..util import get_named_logger, wf_parser  # noqa: ABS101
 
 
 def main(args):
diff --git a/bin/workflow_glue/reheader_samstream.py b/bin/workflow_glue/wfg_helpers/reheader_samstream.py
similarity index 85%
rename from bin/workflow_glue/reheader_samstream.py
rename to bin/workflow_glue/wfg_helpers/reheader_samstream.py
index 1d6528a..1c0344a 100644
--- a/bin/workflow_glue/reheader_samstream.py
+++ b/bin/workflow_glue/wfg_helpers/reheader_samstream.py
@@ -52,9 +52,10 @@ the PG header. This script is a little overkill but attempts to be robust
 with handling PG collisions and more obviously encapsulates reheadering
 behaviour, and leaves some room to do more clever things as necessary.
 """
+from shutil import copyfileobj
 import sys
 
-from .util import wf_parser  # noqa: ABS101
+from ..util import wf_parser  # noqa: ABS101
 
 
 class SamHeader:
@@ -106,15 +107,12 @@ class SamHeader:
         if record_type in ["@HD", "@CO", "@SQ"]:
             return record_type, record_data
         elif record_type in ["@RG", "@PG"]:
-            allowed_keys = {
-                "@RG": ["ID", "BC", "CN", "DS", "DT", "FO", "KS", "LB", "PG", "PI", "PL", "PM", "PU", "SM"],  # noqa:E501
-                "@PG": ["ID", "PN", "CL", "PP", "DS", "VN"]
-            }
             for field in record_data.strip().split('\t'):
                 k, v = field.split(':', 1)
-                if k not in allowed_keys[record_type]:
-                    raise Exception(f"{record_type} with bad key '{k}': {record_data}")
-                record[k] = v
+                if len(k) == 2 and k[0].isalpha() and k[1].isalnum():
+                    record[k] = v
+                else:
+                    raise Exception(f"{record_type} with invalid tag: '{k}'")
             if "ID" not in record:
                 raise Exception(f"{record_type} with no ID: {record_data}")
             return record_type, record
@@ -273,9 +271,29 @@ def reheader_samstream(header_in, stream_in, stream_out, args):
             break
         sh.add_line(line)
 
-    # Pass through the rest of the alignments
-    for line in stream_in:
-        stream_out.write(line)
+    # Pass through the rest of the alignments.
+    # I toyed with a few ways of doing this:
+    #   - A trivial iter over the input file was slow. presumably as we incurred some
+    #     overhead calling read() and write() and decoding more than other methods.
+    #   - os.read/write avoids dealing with higher level python read/write but requires
+    #     file descriptors which rules out non-file-like objects. this made testing more
+    #     annoying as StringIO does not have a file descriptor. we could have mocked fds
+    #     but i was not happy with the discrepancy between real and test execution.
+    #   - copyfileobj with the stream_in.buffer would also avoid some of the higher
+    #     level text handling but would require all tests to provide inputs that have
+    #     an underlying binary buffer. it was also not possible to seek the buffer to
+    #     the position of the text stream as we've used next() to iterate over the
+    #     header lines, fixing this would have required rewriting of the header
+    #     handling or keeping track of the position in the stream ourselves which
+    #     just seemed unncessary overkill given how we expect this program to be used.
+    # copyfileobj on the text streams is more efficient than merely iterating the file
+    # and dumping the lines out and seems to do the job. this keeps the code and tests
+    # simple with minimal additional cost to performance. i anticipate any overhead of
+    # this program will be dwarfed by that of minimap2/samtools sort anyway.
+    # increasing the buffer size gave worse performance in my limited testing so we
+    # leave it as the default here.
+    copyfileobj(stream_in, stream_out)
+
     # If there were no alignments, we won't have hit the != @ case in the first stdin,
     # and we won't have written the header out. Write a header if we haven't already.
     if not wrote_header:
diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy
index 81fdc2e..8998d2a 100644
--- a/lib/NfcoreSchema.groovy
+++ b/lib/NfcoreSchema.groovy
@@ -191,13 +191,14 @@ class NfcoreSchema {
         if (unexpectedParams.size() > 0) {
             Map colors = NfcoreTemplate.logColours(params.monochrome_logs)
             println ''
-            def warn_msg = 'Found unexpected parameters:'
+            def error_msg = 'Found unexpected parameters:'
             for (unexpectedParam in unexpectedParams) {
-                warn_msg = warn_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}"
+                error_msg = error_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}"
             }
-            log.warn warn_msg
-            log.info "${colors.dim}- Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}"
+            log.error error_msg
+            log.info "${colors.dim}- Ignore unexpected params: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}"
             println ''
+            has_error = true
         }
 
         if (has_error) {
diff --git a/lib/common.nf b/lib/common.nf
index 745b892..82a9d12 100644
--- a/lib/common.nf
+++ b/lib/common.nf
@@ -9,7 +9,7 @@ process getParams {
     output:
         path "params.json"
     script:
-        def paramsJSON = new JsonBuilder(params).toPrettyString()
+        def paramsJSON = new JsonBuilder(params).toPrettyString().replaceAll("'", "'\\\\''")
     """
     # Output nextflow params object to JSON
     echo '$paramsJSON' > params.json
diff --git a/lib/ingress.nf b/lib/ingress.nf
index 72abfb5..d12c041 100644
--- a/lib/ingress.nf
+++ b/lib/ingress.nf
@@ -2,12 +2,6 @@ import java.nio.file.NoSuchFileException
 
 import ArgumentParser
 
-enum InputType {
-    SingleFile,
-    TopLevelDir,
-    DirWithSubDirs,
-}
-
 N_OPEN_FILES_LIMIT = 128
 
 
@@ -23,6 +17,28 @@ def is_target_file(Path file, List extensions) {
 }
 
 
+/**
+* Check if a file path is flagged for exclusion.
+*
+* @param p: path to the file in question
+* @param margs: map of ingress args
+* @return: boolean whether the file should be excluded by ingress
+*/
+def is_excluded(Path p, Map margs) {
+    // filter target files for unclassified and failed directories
+    def this_path_parts = p.parent.toString().split(File.separator);
+    def this_unclassified = this_path_parts.contains("unclassified")
+    def this_fail = this_path_parts.contains("pod5_fail") || this_path_parts.contains("bam_fail") || this_path_parts.contains("fastq_fail")
+
+    def filter_unclassified = this_unclassified && !margs.analyse_unclassified
+    def filter_fail = this_fail && !margs.analyse_fail
+
+    // this function exits true and this file will be flagged for exclusion if
+    // any of the exclusion criteria is true
+    filter_unclassified || filter_fail
+}
+
+
 /**
  * Take a channel of the shape `[meta, reads, path-to-stats-dir | null]` (or
  * `[meta, [reads, index], path-to-stats-dir | null]` in the case of XAM) and extract the
@@ -40,10 +56,10 @@ def add_run_IDs_and_basecall_models_to_meta(ch, boolean allow_multiple_basecall_
     // as `ingressed_run_ids`
     ch = ch | map { meta, reads, stats ->
         if (stats) {
-            run_ids = stats.resolve("run_ids").splitText().collect { it.strip() }
+            def run_ids = stats.resolve("run_ids").splitText().collect { it.strip() }
             ingressed_run_ids += run_ids
 
-            basecall_models = \
+            def basecall_models = \
                 stats.resolve("basecallers").splitText().collect { it.strip() }
             // check if we got more than one basecall model and set reads + stats to
             // `null` for that sample unless `allow_multiple_basecall_models`
@@ -270,7 +286,7 @@ def xam_ingress(Map arguments)
 
     // check BAM headers to see if any samples are uBAM
     ch_result = input.dirs
-    | map { meta, path -> [meta, get_target_files_in_dir(path, xam_extensions)] }
+    | map { meta, path -> [meta, get_target_files_in_dir(path, xam_extensions, margs)] }
     | mix(input.files)
     | map{
         // If there is more than one BAM in each folder we ignore
@@ -576,7 +592,7 @@ process fastcat {
 
     # Save file as compressed fastq
     fastcat \
-        -s ${meta["alias"]} \
+        -s '${meta["alias"].replaceAll("'","'\\\\''")}' \
         -f fastcat_stats/per-file-stats.tsv \
         -i fastcat_stats/per-file-runids.tsv \
         -l fastcat_stats/per-file-basecallers.tsv \
@@ -767,18 +783,18 @@ def watch_path(Path input, Map margs, ArrayList extensions) {
     // directory and (ii) files being generated in sub-directories. If we find files of
     // both kinds, throw an error.
     if (input.isFile()) {
-        error "Input ($input) must be a directory when using `watch_path`."
+        error "Input ($input) must be a folder when using `watch_path`."
     }
     // get existing target files first (look for relevant files in the top-level dir and
     // all sub-dirs)
     def ch_existing_input = Channel.fromPath(input)
     | concat(Channel.fromPath("$input/*", type: 'dir'))
-    | map { get_target_files_in_dir(it, extensions) }
+    | map { get_target_files_in_dir(it, extensions, margs, recursive=false) }
     | flatten
     // now get channel with files found by `watchPath`
     def ch_watched = Channel.watchPath("$input/**").until { it.name.startsWith('STOP') }
     // only keep target files
-    | filter { is_target_file(it, extensions) }
+    | filter { is_target_file(it, extensions) && !is_excluded(it, margs) }
     // merge the channels
     ch_watched = ch_existing_input | concat(ch_watched)
     // check if input is as expected; start by throwing an error when finding files in
@@ -788,7 +804,7 @@ def watch_path(Path input, Map margs, ArrayList extensions) {
     | map {
         String input_type = (it.parent == input) ? "top-level" : "sub-dir"
         if (prev_input_type && (input_type != prev_input_type)) {
-            error "`watchPath` found input files in the top-level directory " +
+            error "`watchPath` found input files in the top-level folder " +
                 "as well as in sub-directories."
         }
         // if file is in a sub-dir, make sure it's not a sub-sub-dir
@@ -798,7 +814,7 @@ def watch_path(Path input, Map margs, ArrayList extensions) {
         }
         // we also don't want files in the top-level dir when we got a sample sheet
         if ((input_type == "top-level") && margs["sample_sheet"]) {
-            error "`watchPath` found input files in top-level directory even though " +
+            error "`watchPath` found input files in top-level folder even though " +
                 "a sample sheet was provided ('${margs["sample_sheet"]}')."
         }
         prev_input_type = input_type
@@ -819,7 +835,7 @@ def watch_path(Path input, Map margs, ArrayList extensions) {
             Map sample_sheet_entry = sample_sheet_map[barcode]
             // throw error if the barcode was not in the sample sheet
             if (!sample_sheet_entry) {
-                error "Sub-directory $barcode was not found in the sample sheet."
+                error "Sub-folder $barcode was not found in the sample sheet."
             }
             [create_metamap(sample_sheet_entry), file_path]
         }
@@ -908,6 +924,7 @@ Map parse_arguments(String func_name, Map arguments, Map extra_kwargs=[:]) {
         "sample": null,
         "sample_sheet": null,
         "analyse_unclassified": false,
+        "analyse_fail": false,
         "stats": true,
         "required_sample_types": [],
         "watch_path": false,
@@ -939,65 +956,202 @@ Map parse_arguments(String func_name, Map arguments, Map extra_kwargs=[:]) {
 def get_valid_inputs(Map margs, ArrayList extensions){
     log.info "Searching input for $extensions files."
     Path input
+
+    // check input path exists
     try {
         input = file(margs.input, checkIfExists: true)
     } catch (NoSuchFileException e) {
         error "Input path $margs.input does not exist."
     }
+
     // declare resulting input channel
     def ch_input
+
     // run `watchPath` if requested
     if (margs["watch_path"]) {
         ch_input = watch_path(input, margs, extensions)
+
+    // otherwise, easy case is this a file?
+    } else if (input.isFile()) {
+        if (!is_target_file(input, extensions)) {
+            error "Input file is not of required file type."
+        }
+        ch_input = Channel.of(
+            [create_metamap([alias: margs["sample"] ?: input.simpleName]), input])
+
+    // before we handle a directory, check the path is not something ...weird
+    } else if (!input.isDirectory()){
+        error "Input $input appears to be neither a file nor a folder."
+
+    // we're a directory and one of three cases applies
+    // (i)  a single directory with only target files (old case 2)
+    // (ii) multiple directories with only target files (eg. demultiplexed barcodes - old case 3)
+    // (iii) an arbitrarily nested directory layout (eg. MinKNOW experiment - new case 4)
     } else {
-        // check which of the allowed input types (single file, top-lvl dir, dir with
-        // sub-dirs) we got
-        InputType input_type = determine_input_type(
-            input, extensions, margs.analyse_unclassified
-        )
-        // handle case of `input` being a single file
-        if (input_type == InputType.SingleFile) {
-            ch_input = Channel.of(
-                [create_metamap([alias: margs["sample"] ?: input.simpleName]), input])
-        } else if (input_type == InputType.TopLevelDir) {
-            // input is a directory containing target files
-            ch_input = Channel.of(
-                [create_metamap([alias: margs["sample"] ?: input.baseName]), input])
-        } else {
-            // input is a directory with sub-directories (e.g. barcodes) containing
-            // target files --> find these sub-directories
-            ArrayList sub_dirs_with_target_files = file(
-                input.resolve('*'), type: "dir"
-            ).findAll { get_target_files_in_dir(it, extensions) }
-            // remove directories called 'unclassified' unless otherwise specified
-            if (!margs.analyse_unclassified) {
-                sub_dirs_with_target_files = sub_dirs_with_target_files.findAll {
-                    it.baseName != "unclassified"
+        // work out what we're dealing with:
+        // - iterate over all target files in the tree
+        // - ignoring (or including) unclassified and failures as required
+        // - check the depth of each file (by counting the number of components in its path)
+        // - all files must have the same depth
+        // - if all files also have the same depth as the input dir
+        //   then this is a simple case of a single directory of files
+        // - if all files have depth + 1, then this is the case 3 case
+        Boolean is_singleplex_dir = true
+        Boolean is_multiplex_dir = true
+        Integer input_depth = input.toString().count(File.separator)
+        String this_parent
+        String first_parent
+        Integer this_depth
+        Integer first_depth
+
+        // enumerate all valid files and check their depths
+        // this is not responsible for returning the list of files
+        // this is done regardless of case, as singleplex, multiplex and experiment dirs have the same requirement
+        ArrayList all_files = get_target_files_in_dir(input, extensions, margs)
+        .each {
+            this_parent = it.parent.toString()
+            this_depth = this_parent.count(File.separator)
+            if (first_depth == null) {
+                first_depth = this_depth
+                first_parent = this_parent
+            }
+            else {
+                // this file has different depth from first file - abort accordingly
+                if (this_depth != first_depth) {
+                    error "Found files at different levels in your input folder:\n* ${this_parent}\n* ${first_parent}\n\nAll files in the input folder must be at the same folder level. Please reorganise and try again."
                 }
             }
-            // filter based on sample sheet in case one was provided
-            if (margs.sample_sheet) {
+            // this file has different depth from the input directory path - we're not in the single directory of files case
+            if (this_depth != input_depth) {
+                is_singleplex_dir = false
+            }
+            // this file has different depth from the input directory path + 1 - we're not in the multiplex directory case
+            if (this_depth != (input_depth + 1)) {
+                is_multiplex_dir = false
+            }
+        }
+
+        // if we are neither singleplex (case 2), nor multiplex (case 3), we must be an experiment dir (case 4)
+        // a sample sheet or sample name is required to ensure we ingest the right data
+        Boolean is_experimental_dir = !(is_singleplex_dir || is_multiplex_dir)
+        if (is_experimental_dir) {
+            if (!(margs.sample_sheet || margs.sample)) {
+                error "Sample sheet or sample name must be provided."
+            }
+            if (extensions[0] == ".fastq") {
+                // nextflow is used to manage the BAM files sent to bamstats/xam_ingress
+                // however, fastcat is used to manage FASTQ files directly, meaning it does not support analyse_unclassified,analyse_fail in the same way
+                // we'll avoid support for it for now
+                // see CW-5613
+                error "FASTQ input not currently supported when ingressing MinKNOW experiment folder."
+            }
+        }
+
+        // define string to re-use in error messages below
+        String target_files_str = \
+            "${extensions.collect{'\'' + it + '\''}.join(' / ')}"
+
+        // cry for help if there are no target files
+        if (all_files.size() == 0) {
+            error "No valid files ending in ${target_files_str} found in input folder '${input}'."
+
+        // input is a simple single top level directory containing target files
+        } else if (is_singleplex_dir) {
+            ch_input = Channel.of(
+                [create_metamap([alias: margs["sample"] ?: input.baseName]), input])
+
+        // otherwise we're looking at a directory tree
+        } else {
+            // input is a directory with sub-directories (e.g. barcodes/aliases)
+            // with zero or more further sub-directories
+            // resolve with * to find the first level subdirs and filter out
+            //   any entries that do not have any target files
+            ArrayList sub_dirs_with_target_files = file(
+                input.resolve('*'), type: "dir"
+            ).findAll { get_target_files_in_dir(it, extensions, margs) }
+
+            // filter ingressed dirs to named sample - no sample sheet
+            if (margs.sample && !margs.sample_sheet) {
+                ch_input = Channel.fromPath(sub_dirs_with_target_files).map {
+                    if(it.baseName == margs.sample) {
+                        [create_metamap([alias: it.baseName, barcode: it.baseName]), it]
+                    }
+                    else {
+                        log.warn "Ignoring $it.baseName: Found in input folder but does not match sample name provided ($margs.sample)."
+                    }
+                }
+            }
+            else if (margs.sample_sheet) {
                 // get channel of entries in the sample sheet
                 def ch_sample_sheet = get_sample_sheet(
                     file(margs.sample_sheet), margs.required_sample_types
                 )
-                // get the union of both channels (missing values will be replaced with
-                // `null`)
-                def ch_union = Channel.fromPath(sub_dirs_with_target_files).map {
-                    [it.baseName, it]
-                }.join(ch_sample_sheet.map{[it.barcode, it]}, remainder: true)
+
+                // Divide samples into barcoded and aliased,
+                // we'll join these to the sample sheet individually
+                ch_samples = Channel.fromPath(sub_dirs_with_target_files)
+                | map { [it.baseName, it] }
+                | branch { basename, path ->
+                    barcoded: basename.startsWith("barcode")
+                    aliased: true
+                }
+
+                // Join barcoded samples to sample sheet, remove entries that do not match to sheet and warn accordingly
+                // after join. Yields [basename, path (if joined), alias, sample_sheet_row] for samples on disk and sample sheet,
+                // otherwise yields [basename, path, null] for samples missing a sample sheet entry, we'll prune these out
+                // by looking for a null alias (ie. no sample sheet entry) to prevent a join error on ch_union below.
+                ch_samples_barcoded = ch_samples.barcoded
+                    | join(ch_sample_sheet.map{ [it.barcode, it.alias, it] }, remainder:true)
+                    | map {
+                        if (it[2]) { it }
+                        else { log.warn "Ignoring ${it[0]}: Found in input folder but sample sheet has no such entry." }
+                    }
+                // repeat the above for aliased samples
+                ch_samples_aliased = ch_samples.aliased
+                    | join(ch_sample_sheet.map{ [it.alias, it.alias, it] }, remainder:true)
+                    | map {
+                        if (it[2]) { it }
+                        else { log.warn "Ignoring ${it[0]}: Found in input folder but sample sheet has no such entry." }
+                    }
+
+                // It is now safe to join (on alias) the barcode and alias samples together as we've removed entries that conflict with the sample sheet.
+                // The ch_union channel will now have an element for each row of the sample sheet
+                // combining the barcode and alias information and any paths for either that were matched on disk
+                ch_union = ch_samples_barcoded.join(ch_samples_aliased, by:2)
+
                 // after joining the channels, there are three possible cases:
-                // (i) valid input path and sample sheet entry are both present
+                // (i) valid input path for ONE of barcode and alias, and its sample sheet entry is present
+                //      --> we'll emit `[metamap-from-sample-sheet-entry, path]`
                 // (ii) there is a sample sheet entry but no corresponding input dir
                 //      --> we'll emit `[metamap-from-sample-sheet-entry, null]`
-                // (iii) there is a valid path, but the sample sheet entry is missing
-                //      --> drop this entry and print a warning to the log
-                ch_input = ch_union.map {barcode, path, sample_sheet_entry ->
-                    if (sample_sheet_entry) {
+                // (iii) valid input path for BOTH barcode and alias, and its sample sheet entry are present
+                //      --> a directory for both the barcode and alias have been provided
+                //          and we don't know which to pick, so we'll raise an error for this conflict
+                // * sample_sheet_entry will be set here as we've filtered out those cases above
+                // * _alias and _sample_sheet_entry and merely unused dupes of alias and sample_sheet_entry due to the ch_union join
+                ch_input = ch_union.map {alias, barcode, barcode_path, sample_sheet_entry, _alias, alias_path, _sample_sheet_entry ->
+                    def path = null
+                    if (barcode_path && alias_path){
+                        error "Found conflicting folders and cannot ingress both sample folder '$alias' and barcode folder '$barcode' for same sample sheet row."
+                    }
+                    else if (barcode_path || alias_path) {
+                        path = barcode_path ?: alias_path
+                    }
+
+                    if (!path) {
+                        log.warn "Ignoring $alias: Found in sample sheet but a corresponding sample folder was not found in the input folder."
+                    }
+                    if(margs.sample) {
+                        if (alias == margs.sample || barcode == margs.sample) {
+                            [create_metamap(sample_sheet_entry), path]
+                        }
+                        else if (path) {
+                            // only emit "found in input folder" if a path exists
+                            log.warn "Ignoring $alias: Found in input folder and sample sheet, but does not match sample name provided ($margs.sample)."
+                        }
+                    }
+                    else {
                         [create_metamap(sample_sheet_entry), path]
-                    } else {
-                        log.warn "Input directory '$barcode' was found, but sample " +
-                            "sheet '$margs.sample_sheet' has no such entry."
                     }
                 }
             } else {
@@ -1008,19 +1162,23 @@ def get_valid_inputs(Map margs, ArrayList extensions){
             }
         }
     }
-    // finally, we "unwrap" directories containing only a single file and then split the
-    // results channel into the three different output types (sample sheet entries
-    // without corresponding barcodes -- i.e. with `path == null`, single files, and
-    // dirs with multiple files)
-    def ch_branched_results = ch_input.map { meta, path ->
+    // unwrap folders containing a single target file into a channel for just that file
+    // then return a branched channel containing:
+    // * missing - indicating sample sheet entries that were not matched to the input
+    //             directory, the meta is populated but the path is null
+    // * files - single file inputs (including those from a directory with a single file)
+    // * dirs - directory inputs in need of munging downstream
+    def ch_branched_results = ch_input
+    | map { meta, path ->
         if (path && path.isDirectory()) {
-            List fq_files = get_target_files_in_dir(path, extensions)
+            List fq_files = get_target_files_in_dir(path, extensions, margs)
             if (fq_files.size() == 1) {
                 path = fq_files[0]
             }
         }
         [meta, path]
-    } .branch { meta, path ->
+    }
+    | branch { meta, path ->
         missing: !path
         files: path.isFile()
         dirs: path.isDirectory()
@@ -1028,68 +1186,6 @@ def get_valid_inputs(Map margs, ArrayList extensions){
     return ch_branched_results
 }
 
-/**
- * Determine which of the allowed categories (single file, top-level directory, or
- * directory with sub-directory) an input path belongs to.
- *
- * @param margs: parsed arguments (see `fastq_ingress()` or `xam_ingress()` for details)
- * @param extensions: list of valid extensions for the target file type
- * @return: input type represented as an instance of the `InputType` enum
- */
-InputType determine_input_type(
-    Path input, ArrayList extensions, boolean analyse_unclassified
-) {
-    if (input.isFile()) {
-        if (!is_target_file(input, extensions)) {
-            error "Input file is not of required file type."
-        }
-        return InputType.SingleFile
-    } else if (!input.isDirectory()){
-        error "Input $input appears to be neither a file nor a directory."
-    }
-    // `input` is a directory --> we accept two cases: (i) a top-level directory with
-    // target files and no sub-directories or (ii) a directory with one layer of
-    // sub-directories containing target files. First, check if the directory contains
-    // target files and find potential sub-directories (and sub-dirs with target files;
-    // note that these lists can be empty)
-    boolean dir_has_target_files = get_target_files_in_dir(input, extensions)
-    ArrayList sub_dirs = file(input.resolve('*'), type: "dir")
-    ArrayList sub_dirs_with_target_files = sub_dirs.findAll {
-        get_target_files_in_dir(it, extensions)
-    }.findAll { it.baseName != "unclassified" || analyse_unclassified }
-
-    // define string to re-use in error messages below
-    String target_files_str = \
-        "target files (ending in ${extensions.collect{'\'' + it + '\''}.join(' / ')})"
-
-    // check for target files in the top-level dir; if there are any, make sure there
-    // are no sub-directories containing target files
-    if (dir_has_target_files) {
-        if (sub_dirs_with_target_files) {
-            error "Input directory '$input' cannot contain $target_files_str " +
-                "and also sub-directories with such files."
-        }
-        return InputType.TopLevelDir
-    }
-
-    // no target files in the top-level dir --> make sure there were sub-dirs with
-    // target files
-    if (!sub_dirs_with_target_files) {
-        error "Input directory '$input' must contain either $target_files_str " +
-            "or sub-directories containing such files (no more than one layer deep)."
-    }
-    // we don't allow sub-sub-directories with target files
-    if (sub_dirs.any {
-        ArrayList subsubdirs = file(it.resolve('*'), type: "dir")
-        subsubdirs.any { get_target_files_in_dir(it, extensions) }
-    }) {
-        error "Input directory '$input' cannot contain more " +
-            "than one level of sub-directories with $target_files_str."
-    }
-    return InputType.DirWithSubDirs
-}
-
-
 /**
  * Create a map that contains at least these keys: `[alias, barcode, type]`.
  * `alias` is required, `barcode` and `type` are filled with default values if
@@ -1116,14 +1212,18 @@ Map create_metamap(Map arguments) {
 
 
 /**
- * Get the target files in the directory (non-recursive).
+ * Get all target files below this directory.
  *
  * @param dir: path to the target directory
  * @param extensions: list of valid extensions for the target file type
+ * @param margs: ingress margs
  * @return: list of found target files
  */
-ArrayList get_target_files_in_dir(Path dir, ArrayList extensions) {
-    file(dir.resolve("*")).findAll { is_target_file(it, extensions) }
+ArrayList get_target_files_in_dir(Path dir, ArrayList extensions, Map margs, Boolean recursive = true) {
+    String resolver = recursive ? "**" : "*"
+    file(dir.resolve(resolver)).findAll {
+        is_target_file(it, extensions) && !is_excluded(it, margs)
+    }
 }
 
 
diff --git a/nextflow.config b/nextflow.config
index 36e9bf0..aadee65 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -73,6 +73,7 @@ params {
     plot_gffcmp_stats = true
 
     disable_ping = false
+    store_dir = null
 
     // de options
     de_analysis = false
@@ -95,7 +96,7 @@ params {
       ]
     agent = null
     container_sha = "shac733d952a14257cf3c5c5d5d44c6aed84d5fe5a1"
-	common_sha = "shaabceef445fb63214073cbf5836fdd33c04be4ac7"
+	common_sha = "sha9ef2f4e4585c4ce6a604616e77185077551abf50"
     }
 }
 
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 9949038..8d44fa2 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -316,6 +316,10 @@
                     "fa_icon": "fas fa-question-circle",
                     "hidden": true
                 },
+                "store_dir": {
+                    "type": "string",
+                    "hidden": true
+                },
                 "disable_ping": {
                     "type": "boolean",
                     "default": false,