Template update 5.6.0 [CW-6070]

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
         always_run: true
         pass_filenames: false
         additional_dependencies:
-          - epi2melabs==0.0.57
+          - epi2melabs==0.0.58
   - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [Unreleased]
+### Changed
+- Updated to wf-template v5.6.0, changing:
+    - Reduce verbosity of debug logging from fastcat which can occasionally occlude errors found in FASTQ files during ingress.
+    - Log banner art to say "EPI2ME" instead of "EPI2ME Labs" to match current branding. This has no effect on the workflow outputs.
+
+### Fixed
+- Updated to wf-template v5.6.0, fixing:
+    - dacite.exceptions.WrongTypeError during report generation when barcode is null.
+    - Sequence summary read length N50 incorrectly displayed minimum read length, it now correctly shows the N50.
+    - Sequence summary component alignment and coverage plots failed to plot under some conditions.
+
 ## [v1.7.0]
 ### Changed
 - `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased.

README.md

@@ -48,7 +48,7 @@ therefore Nextflow will need to be
 installed before attempting to run the workflow.
 
 The workflow can currently be run using either
-[Docker](https://www.docker.com/products/docker-desktop)
+[Docker](https://docs.docker.com/get-started/)
 or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html)
 to provide isolation of the required software.
 Both methods are automated out-of-the-box provided

bin/workflow_glue/models/common.py

@@ -1,9 +1,69 @@
 """Common model classes used across all workflows."""
 from dataclasses import asdict, dataclass, field
-from decimal import Decimal
 from enum import Enum
 import json
-from typing import Any, Dict, List
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..util import get_named_logger  # noqa: ABS101
+
+logger = get_named_logger("Models")
+
+
+@dataclass
+class WorkflowBaseModel:
+    """Common things for stuff in the model."""
+
+    def get(
+        self,
+        field_name: str,
+        title: bool = True,
+        **kwargs
+    ):
+        """Get reportable field tuple."""
+        field_info = self.__dataclass_fields__.get(field_name)
+        # provide an empty string default title to minimise drama
+        field_title = field_info.metadata.get("title", "")
+        value = self.get_reportable_value(field_name=field_name, **kwargs)
+        if title:
+            return (field_title, value)
+        return value
+
+    def get_reportable_value(
+            self,
+            field_name: str,
+            *,
+            decimal_places: int = None,
+            default_value: str = "N/A") -> Optional[str]:
+        """Get the value of a value and make it reportable."""
+        # Get the field info using the field name
+        field_info = self.__dataclass_fields__.get(field_name)
+        if field_info is None:
+            raise AttributeError(
+                f"{field_name!r} is not a field on {self.__class__.__name__}"
+            )
+
+        value = getattr(self, field_name)
+
+        if value is None:
+            return default_value
+
+        if isinstance(value, (int, float)):
+            if decimal_places:
+                value = round(value, decimal_places)
+            if value < 0.0001 or value > 99999999:
+                value = f"{value:.2E}"
+        else:
+            if decimal_places:
+                raise TypeError(
+                    "decimal_places is not a supported argument for a non-numeric.")
+
+        unit = field_info.metadata.get('unit')
+
+        if unit:
+            return f"{value} {unit}"
+
+        return str(value)
 
 
 class SampleType(str, Enum):
@@ -86,10 +146,6 @@ class Sample:
         metadata={
             "title": "Sample alias",
             "description": "The alias for the sample given by the user"})
-    barcode: str = field(
-        metadata={
-            "title": "Sample barcode",
-            "description": "The physical barcode assigned to the sample"})
     sample_type: SampleType = field(
         metadata={
             "title": "Sample type",
@@ -98,10 +154,15 @@ class Sample:
         metadata={
             "title": "Sample pass",
             "description": "If true the sample has passed workflow checks"})
+    barcode: str | None = field(
+        default=None,
+        metadata={
+            "title": "Sample barcode",
+            "description": "The physical barcode assigned to the sample"})
     additional_identifiers: List[SampleIdentifier] = field(
         default_factory=list, metadata={
             "title": "Additional sample identifiers",
-            "description": "Addition identifiers for the sample"})
+            "description": "Additional identifiers for the sample"})
     sample_checks: list[CheckResult] = field(
         default_factory=list, metadata={
             "title": "Sample checks",
@@ -123,9 +184,9 @@ class Sample:
 
     def get_sample_identifier(self, sample_identifier):
         """Get a sample identifier given the identifier name."""
-        for indentifier in self.additional_identifiers:
+        for identifier in self.additional_identifiers:
-            if indentifier.name == sample_identifier:
+            if identifier.name == sample_identifier:
-                return indentifier.value
+                return identifier.value
         raise KeyError("Sample identifier not found")
 
     def set_sample_identifier(self, name, value):
@@ -139,7 +200,42 @@ class Sample:
     def to_json(self, filename):
         """Save class as JSON."""
         with open(filename, 'w') as f:
-            json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder)
+            json.dump(asdict(self), f, default=str, indent=2)
+
+    def get_reportable_qc_status(self, max_criteria=4):
+        """Store global status of the sample and list of QC criteria to show.
+
+        :params max_criteria: Maximum number of criteria to be reported.
+        """
+        # Store global status: pass/ failed
+        qc_global_status = {"status": self.sample_pass, "scope": "QC status"}
+        qc_criteria = []
+        if self.sample_pass:
+            qc_criteria.append(
+                    {"status": self.sample_pass, "scope": "All acceptance criteria met"}
+            )
+        else:
+            # Report failed criteria until a maximum value
+            for qc in self.sample_checks:
+                if not qc.check_pass:  # append criteria if failed
+                    qc_criteria.append(
+                        {
+                            "status": qc.check_pass,
+                            "category": qc.friendly_check_category(),
+                            "scope": qc.friendly_check_name(),
+                        }
+                    )
+            if len(qc_criteria) > max_criteria:
+                # Replace all the failed criteria, with a sentence with the number
+                # instead of listing all of them.
+                # Set status to False as more than max_criteria are failed.
+                qc_criteria = [
+                    {
+                        "status": False,
+                        "scope": f"{len(qc_criteria)} acceptance criteria",
+                    },
+                ]
+        return qc_global_status, qc_criteria
 
 
 @dataclass
@@ -161,7 +257,7 @@ class RunStats:
 
 
 @dataclass
-class WorkflowResult():
+class WorkflowResult(WorkflowBaseModel):
     """
     Definition for results that will be returned by this workflow.
 
@@ -189,18 +285,73 @@ class WorkflowResult():
         default_factory=dict, metadata={
             "title": "Client fields",
             "description": "Arbitrary key-value pairs provided by the client"})
+    versions: dict[str, Any] | None = field(
+        default_factory=dict, metadata={
+            "title": "Analysis tool versions",
+            "description": """Key-value pairs collecting the
+            software used and the corresponding versions"""})
+    params: dict[str, Any] | None = field(
+        default_factory=dict, metadata={
+            "title": "Pertinent parameters",
+            "description": """Key-value pairs with the
+            options chosen by the user"""})
+
+    def load_client_fields(self, filename):
+        """Load client fields."""
+        with open(filename) as f:
+            try:
+                client_fields = json.loads(f.read())
+                # convert any lists into strings for display
+                for key, value in client_fields.items():
+                    if isinstance(value, list):
+                        client_fields[key] = ', '.join(value)
+            except json.decoder.JSONDecodeError:
+                client_fields = {"error": "Error parsing client fields file."}
+
+        self.client_fields = client_fields
+        return self.client_fields
+
+    def load_params(self, params_json, keep=None):
+        """Create a workflow params dict."""
+        params_json = Path(params_json)
+        if keep is None:
+            keep = []
+        if not params_json.is_file():
+            raise FileNotFoundError(f"No such file: {params_json}")
+        with open(params_json, "r") as f:
+            try:
+                params_dict = json.loads(f.read())
+                self.params = {
+                    k: v for k, v in params_dict.items() if k in set(keep)
+                }
+                return self.params
+            except ValueError:
+                raise ValueError(f"Invalid JSON file: {params_json}")
+
+    def load_versions(self, versions_path):
+        """Create a version list of dict."""
+        versions_path = Path(versions_path)
+        if not versions_path.exists():
+            raise FileNotFoundError(f"No such file: {versions_path}")
+
+        if versions_path.is_dir():
+            version_files = [
+                vp for vp in versions_path.iterdir() if vp.is_file()
+            ]
+        elif versions_path.is_file():
+            version_files = [versions_path]
+        else:
+            raise IOError(f"{versions_path} should be either a directory or a file")
+        for fname in version_files:
+            versions = {}
+            with open(fname, "r", encoding="utf-8") as fh:
+                for line in fh.readlines():
+                    name, version = line.strip().split(",")
+                    versions[name] = version
+        self.versions = versions
+        return self.versions
 
     def to_json(self, filename):
         """Save class as JSON."""
         with open(filename, 'w') as f:
-            json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder)
+            json.dump(asdict(self), f, default=str, indent=2)
-
-
-class DecimalEncoder(json.JSONEncoder):
-    """This should probably be moved."""
-
-    def default(self, obj):
-        """Override the default method to handle Decimal objects."""
-        if isinstance(obj, Decimal):
-            return float(obj)
-        return super().default(obj)

docs/04_install_and_run.md

@@ -9,7 +9,7 @@ therefore Nextflow will need to be
 installed before attempting to run the workflow.
 
 The workflow can currently be run using either
-[Docker](https://www.docker.com/products/docker-desktop)
+[Docker](https://docs.docker.com/get-started/)
 or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html)
 to provide isolation of the required software.
 Both methods are automated out-of-the-box provided

lib/NfcoreTemplate.groovy

@@ -326,14 +326,16 @@ class NfcoreTemplate {
         String workflow_version = version(workflow)
         String.format(
             """
-            ${colors.igreen}||||||||||   ${colors.reset}${colors.dim}_____ ____ ___ ____  __  __ _____      _       _
+            ${colors.igreen}||||||||||   ${colors.reset}${colors.dim}_____ ____ ___ ____  __  __ _____
-            ${colors.igreen}||||||||||  ${colors.reset}${colors.dim}| ____|  _ \\_ _|___ \\|  \\/  | ____|    | | __ _| |__  ___
+            ${colors.igreen}||||||||||  ${colors.reset}${colors.dim}| ____|  _ \\_ _|___ \\|  \\/  | ____|
-            ${colors.yellow}|||||       ${colors.reset}${colors.dim}|  _| | |_) | |  __) | |\\/| |  _| _____| |/ _` | '_ \\/ __|
+            ${colors.yellow}|||||       ${colors.reset}${colors.dim}|  _| | |_) | |  __) | |\\/| |  _|
-            ${colors.yellow}|||||       ${colors.reset}${colors.dim}| |___|  __/| | / __/| |  | | |__|_____| | (_| | |_) \\__ \\
+            ${colors.yellow}|||||       ${colors.reset}${colors.dim}| |___|  __/| | / __/| |  | | |__
-            ${colors.iblue}||||||||||  ${colors.reset}${colors.dim}|_____|_|  |___|_____|_|  |_|_____|    |_|\\__,_|_.__/|___/
+            ${colors.iblue}||||||||||  ${colors.reset}${colors.dim}|_____|_|  |___|_____|_|  |_|_____|
             ${colors.iblue}||||||||||  ${colors.reset}${colors.bold}${workflow_name} ${workflow_version}${colors.reset}
             ${NfcoreTemplate.dashedLine(monochrome_logs)}
             """.stripIndent()
         )
     }
 }
+
+

lib/Pinguscript.groovy

@@ -48,13 +48,6 @@ class Pinguscript {
         String paramsJSON = new JsonBuilder(params).toPrettyString()
         def params_data = new JsonSlurper().parseText(paramsJSON)
 
-        // hostname
-        def host = null
-        try {
-            host = InetAddress.getLocalHost().getHostName()
-        }
-        catch(Exception e) {}
-
         // OS
         // TODO check version on WSL
         def opsys = System.properties['os.name'].toLowerCase()
@@ -103,7 +96,7 @@ class Pinguscript {
         body_json \
             "tracking_id": [
                 "msg_id": UUID.randomUUID().toString(),
-                "version": "3.0.0"
+                "version": "3.0.1"
             ],
             "source": "workflow",
             "event": event,
@@ -123,7 +116,6 @@ class Pinguscript {
             ],
             "env": [
                 "user": user, // placeholder for any future okta
-                "hostname": host,
                 "os": [
                     "name": opsys,
                     "version": opver

nextflow.config

@@ -96,7 +96,7 @@ params {
       ]
     agent = null
     container_sha = "shac733d952a14257cf3c5c5d5d44c6aed84d5fe5a1"
-	common_sha = "sha9ef2f4e4585c4ce6a604616e77185077551abf50"
+	common_sha = "sha1c69fd30053aad5d516e9567b3944384325a0fee"
     }
 }
 

nextflow_schema.json

@@ -323,7 +323,12 @@
                 "disable_ping": {
                     "type": "boolean",
                     "default": false,
-                    "description": "Enable to prevent sending a workflow ping."
+                    "description": "Enable to prevent sending a workflow ping.",
+                    "overrides": {
+                        "epi2mecloud": {
+                            "hidden": true
+                        }
+                    }
                 },
                 "version": {
                     "type": "boolean",