From 18bf1d5b19b31b04f6bb7d798374fed1f87e1e4b Mon Sep 17 00:00:00 2001 From: Neil Horner Date: Tue, 6 May 2025 10:23:01 +0000 Subject: [PATCH] Template update 5.6.0 [CW-6070] --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 13 ++ README.md | 2 +- bin/workflow_glue/models/common.py | 197 +++++++++++++++++++++++++---- docs/04_install_and_run.md | 2 +- lib/NfcoreTemplate.groovy | 12 +- lib/Pinguscript.groovy | 10 +- nextflow.config | 2 +- nextflow_schema.json | 7 +- 9 files changed, 205 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d407af..8003870 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: always_run: true pass_filenames: false additional_dependencies: - - epi2melabs==0.0.57 + - epi2melabs==0.0.58 - repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bad740..88a875e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Changed +- Updated to wf-template v5.6.0, changing: + - Reduce verbosity of debug logging from fastcat which can occasionally occlude errors found in FASTQ files during ingress. + - Log banner art to say "EPI2ME" instead of "EPI2ME Labs" to match current branding. This has no effect on the workflow outputs. + +### Fixed +- Updated to wf-template v5.6.0, fixing: + - dacite.exceptions.WrongTypeError during report generation when barcode is null. + - Sequence summary read length N50 incorrectly displayed minimum read length, it now correctly shows the N50. + - Sequence summary component alignment and coverage plots failed to plot under some conditions. + ## [v1.7.0] ### Changed - `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased. diff --git a/README.md b/README.md index fdb72c0..b9bdae5 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ therefore Nextflow will need to be installed before attempting to run the workflow. The workflow can currently be run using either -[Docker](https://www.docker.com/products/docker-desktop) +[Docker](https://docs.docker.com/get-started/) or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html) to provide isolation of the required software. Both methods are automated out-of-the-box provided diff --git a/bin/workflow_glue/models/common.py b/bin/workflow_glue/models/common.py index 04a217f..65b91fd 100644 --- a/bin/workflow_glue/models/common.py +++ b/bin/workflow_glue/models/common.py @@ -1,9 +1,69 @@ """Common model classes used across all workflows.""" from dataclasses import asdict, dataclass, field -from decimal import Decimal from enum import Enum import json -from typing import Any, Dict, List +from pathlib import Path +from typing import Any, Dict, List, Optional + +from ..util import get_named_logger # noqa: ABS101 + +logger = get_named_logger("Models") + + +@dataclass +class WorkflowBaseModel: + """Common things for stuff in the model.""" + + def get( + self, + field_name: str, + title: bool = True, + **kwargs + ): + """Get reportable field tuple.""" + field_info = self.__dataclass_fields__.get(field_name) + # provide an empty string default title to minimise drama + field_title = field_info.metadata.get("title", "") + value = self.get_reportable_value(field_name=field_name, **kwargs) + if title: + return (field_title, value) + return value + + def get_reportable_value( + self, + field_name: str, + *, + decimal_places: int = None, + default_value: str = "N/A") -> Optional[str]: + """Get the value of a value and make it reportable.""" + # Get the field info using the field name + field_info = self.__dataclass_fields__.get(field_name) + if field_info is None: + raise AttributeError( + f"{field_name!r} is not a field on {self.__class__.__name__}" + ) + + value = getattr(self, field_name) + + if value is None: + return default_value + + if isinstance(value, (int, float)): + if decimal_places: + value = round(value, decimal_places) + if value < 0.0001 or value > 99999999: + value = f"{value:.2E}" + else: + if decimal_places: + raise TypeError( + "decimal_places is not a supported argument for a non-numeric.") + + unit = field_info.metadata.get('unit') + + if unit: + return f"{value} {unit}" + + return str(value) class SampleType(str, Enum): @@ -86,10 +146,6 @@ class Sample: metadata={ "title": "Sample alias", "description": "The alias for the sample given by the user"}) - barcode: str = field( - metadata={ - "title": "Sample barcode", - "description": "The physical barcode assigned to the sample"}) sample_type: SampleType = field( metadata={ "title": "Sample type", @@ -98,10 +154,15 @@ class Sample: metadata={ "title": "Sample pass", "description": "If true the sample has passed workflow checks"}) + barcode: str | None = field( + default=None, + metadata={ + "title": "Sample barcode", + "description": "The physical barcode assigned to the sample"}) additional_identifiers: List[SampleIdentifier] = field( default_factory=list, metadata={ "title": "Additional sample identifiers", - "description": "Addition identifiers for the sample"}) + "description": "Additional identifiers for the sample"}) sample_checks: list[CheckResult] = field( default_factory=list, metadata={ "title": "Sample checks", @@ -123,9 +184,9 @@ class Sample: def get_sample_identifier(self, sample_identifier): """Get a sample identifier given the identifier name.""" - for indentifier in self.additional_identifiers: - if indentifier.name == sample_identifier: - return indentifier.value + for identifier in self.additional_identifiers: + if identifier.name == sample_identifier: + return identifier.value raise KeyError("Sample identifier not found") def set_sample_identifier(self, name, value): @@ -139,7 +200,42 @@ class Sample: def to_json(self, filename): """Save class as JSON.""" with open(filename, 'w') as f: - json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder) + json.dump(asdict(self), f, default=str, indent=2) + + def get_reportable_qc_status(self, max_criteria=4): + """Store global status of the sample and list of QC criteria to show. + + :params max_criteria: Maximum number of criteria to be reported. + """ + # Store global status: pass/ failed + qc_global_status = {"status": self.sample_pass, "scope": "QC status"} + qc_criteria = [] + if self.sample_pass: + qc_criteria.append( + {"status": self.sample_pass, "scope": "All acceptance criteria met"} + ) + else: + # Report failed criteria until a maximum value + for qc in self.sample_checks: + if not qc.check_pass: # append criteria if failed + qc_criteria.append( + { + "status": qc.check_pass, + "category": qc.friendly_check_category(), + "scope": qc.friendly_check_name(), + } + ) + if len(qc_criteria) > max_criteria: + # Replace all the failed criteria, with a sentence with the number + # instead of listing all of them. + # Set status to False as more than max_criteria are failed. + qc_criteria = [ + { + "status": False, + "scope": f"{len(qc_criteria)} acceptance criteria", + }, + ] + return qc_global_status, qc_criteria @dataclass @@ -161,7 +257,7 @@ class RunStats: @dataclass -class WorkflowResult(): +class WorkflowResult(WorkflowBaseModel): """ Definition for results that will be returned by this workflow. @@ -189,18 +285,73 @@ class WorkflowResult(): default_factory=dict, metadata={ "title": "Client fields", "description": "Arbitrary key-value pairs provided by the client"}) + versions: dict[str, Any] | None = field( + default_factory=dict, metadata={ + "title": "Analysis tool versions", + "description": """Key-value pairs collecting the + software used and the corresponding versions"""}) + params: dict[str, Any] | None = field( + default_factory=dict, metadata={ + "title": "Pertinent parameters", + "description": """Key-value pairs with the + options chosen by the user"""}) + + def load_client_fields(self, filename): + """Load client fields.""" + with open(filename) as f: + try: + client_fields = json.loads(f.read()) + # convert any lists into strings for display + for key, value in client_fields.items(): + if isinstance(value, list): + client_fields[key] = ', '.join(value) + except json.decoder.JSONDecodeError: + client_fields = {"error": "Error parsing client fields file."} + + self.client_fields = client_fields + return self.client_fields + + def load_params(self, params_json, keep=None): + """Create a workflow params dict.""" + params_json = Path(params_json) + if keep is None: + keep = [] + if not params_json.is_file(): + raise FileNotFoundError(f"No such file: {params_json}") + with open(params_json, "r") as f: + try: + params_dict = json.loads(f.read()) + self.params = { + k: v for k, v in params_dict.items() if k in set(keep) + } + return self.params + except ValueError: + raise ValueError(f"Invalid JSON file: {params_json}") + + def load_versions(self, versions_path): + """Create a version list of dict.""" + versions_path = Path(versions_path) + if not versions_path.exists(): + raise FileNotFoundError(f"No such file: {versions_path}") + + if versions_path.is_dir(): + version_files = [ + vp for vp in versions_path.iterdir() if vp.is_file() + ] + elif versions_path.is_file(): + version_files = [versions_path] + else: + raise IOError(f"{versions_path} should be either a directory or a file") + for fname in version_files: + versions = {} + with open(fname, "r", encoding="utf-8") as fh: + for line in fh.readlines(): + name, version = line.strip().split(",") + versions[name] = version + self.versions = versions + return self.versions def to_json(self, filename): """Save class as JSON.""" with open(filename, 'w') as f: - json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder) - - -class DecimalEncoder(json.JSONEncoder): - """This should probably be moved.""" - - def default(self, obj): - """Override the default method to handle Decimal objects.""" - if isinstance(obj, Decimal): - return float(obj) - return super().default(obj) + json.dump(asdict(self), f, default=str, indent=2) diff --git a/docs/04_install_and_run.md b/docs/04_install_and_run.md index 49a6b0c..91ea928 100644 --- a/docs/04_install_and_run.md +++ b/docs/04_install_and_run.md @@ -9,7 +9,7 @@ therefore Nextflow will need to be installed before attempting to run the workflow. The workflow can currently be run using either -[Docker](https://www.docker.com/products/docker-desktop) +[Docker](https://docs.docker.com/get-started/) or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html) to provide isolation of the required software. Both methods are automated out-of-the-box provided diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 4c1c9db..908f5f3 100644 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -326,14 +326,16 @@ class NfcoreTemplate { String workflow_version = version(workflow) String.format( """ - ${colors.igreen}|||||||||| ${colors.reset}${colors.dim}_____ ____ ___ ____ __ __ _____ _ _ - ${colors.igreen}|||||||||| ${colors.reset}${colors.dim}| ____| _ \\_ _|___ \\| \\/ | ____| | | __ _| |__ ___ - ${colors.yellow}||||| ${colors.reset}${colors.dim}| _| | |_) | | __) | |\\/| | _| _____| |/ _` | '_ \\/ __| - ${colors.yellow}||||| ${colors.reset}${colors.dim}| |___| __/| | / __/| | | | |__|_____| | (_| | |_) \\__ \\ - ${colors.iblue}|||||||||| ${colors.reset}${colors.dim}|_____|_| |___|_____|_| |_|_____| |_|\\__,_|_.__/|___/ + ${colors.igreen}|||||||||| ${colors.reset}${colors.dim}_____ ____ ___ ____ __ __ _____ + ${colors.igreen}|||||||||| ${colors.reset}${colors.dim}| ____| _ \\_ _|___ \\| \\/ | ____| + ${colors.yellow}||||| ${colors.reset}${colors.dim}| _| | |_) | | __) | |\\/| | _| + ${colors.yellow}||||| ${colors.reset}${colors.dim}| |___| __/| | / __/| | | | |__ + ${colors.iblue}|||||||||| ${colors.reset}${colors.dim}|_____|_| |___|_____|_| |_|_____| ${colors.iblue}|||||||||| ${colors.reset}${colors.bold}${workflow_name} ${workflow_version}${colors.reset} ${NfcoreTemplate.dashedLine(monochrome_logs)} """.stripIndent() ) } } + + diff --git a/lib/Pinguscript.groovy b/lib/Pinguscript.groovy index 4b91236..8b42d34 100644 --- a/lib/Pinguscript.groovy +++ b/lib/Pinguscript.groovy @@ -48,13 +48,6 @@ class Pinguscript { String paramsJSON = new JsonBuilder(params).toPrettyString() def params_data = new JsonSlurper().parseText(paramsJSON) - // hostname - def host = null - try { - host = InetAddress.getLocalHost().getHostName() - } - catch(Exception e) {} - // OS // TODO check version on WSL def opsys = System.properties['os.name'].toLowerCase() @@ -103,7 +96,7 @@ class Pinguscript { body_json \ "tracking_id": [ "msg_id": UUID.randomUUID().toString(), - "version": "3.0.0" + "version": "3.0.1" ], "source": "workflow", "event": event, @@ -123,7 +116,6 @@ class Pinguscript { ], "env": [ "user": user, // placeholder for any future okta - "hostname": host, "os": [ "name": opsys, "version": opver diff --git a/nextflow.config b/nextflow.config index eea1677..5e62528 100644 --- a/nextflow.config +++ b/nextflow.config @@ -96,7 +96,7 @@ params { ] agent = null container_sha = "shac733d952a14257cf3c5c5d5d44c6aed84d5fe5a1" - common_sha = "sha9ef2f4e4585c4ce6a604616e77185077551abf50" + common_sha = "sha1c69fd30053aad5d516e9567b3944384325a0fee" } } diff --git a/nextflow_schema.json b/nextflow_schema.json index 31282da..9e4a2c7 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -323,7 +323,12 @@ "disable_ping": { "type": "boolean", "default": false, - "description": "Enable to prevent sending a workflow ping." + "description": "Enable to prevent sending a workflow ping.", + "overrides": { + "epi2mecloud": { + "hidden": true + } + } }, "version": { "type": "boolean",