Merge branch 'template_update_5.6.0' into 'dev'
Template update 5.6.0 [CW-6070] See merge request epi2melabs/workflows/wf-transcriptomes!213
This commit is contained in:
Neil Horner
commit
1b3f49a1ac
@ -8,7 +8,7 @@ repos:
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
additional_dependencies:
|
||||
- epi2melabs==0.0.57
|
||||
- epi2melabs==0.0.58
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 5.0.4
|
||||
hooks:
|
||||
|
||||
13
CHANGELOG.md
13
CHANGELOG.md
@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
|
||||
## [Unreleased]
|
||||
### Changed
|
||||
- Updated to wf-template v5.6.0, changing:
|
||||
- Reduce verbosity of debug logging from fastcat which can occasionally occlude errors found in FASTQ files during ingress.
|
||||
- Log banner art to say "EPI2ME" instead of "EPI2ME Labs" to match current branding. This has no effect on the workflow outputs.
|
||||
|
||||
### Fixed
|
||||
- Updated to wf-template v5.6.0, fixing:
|
||||
- dacite.exceptions.WrongTypeError during report generation when barcode is null.
|
||||
- Sequence summary read length N50 incorrectly displayed minimum read length, it now correctly shows the N50.
|
||||
- Sequence summary component alignment and coverage plots failed to plot under some conditions.
|
||||
|
||||
## [v1.7.0]
|
||||
### Changed
|
||||
- `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased.
|
||||
|
||||
@ -48,7 +48,7 @@ therefore Nextflow will need to be
|
||||
installed before attempting to run the workflow.
|
||||
|
||||
The workflow can currently be run using either
|
||||
[Docker](https://www.docker.com/products/docker-desktop)
|
||||
[Docker](https://docs.docker.com/get-started/)
|
||||
or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html)
|
||||
to provide isolation of the required software.
|
||||
Both methods are automated out-of-the-box provided
|
||||
|
||||
@ -1,9 +1,69 @@
|
||||
"""Common model classes used across all workflows."""
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..util import get_named_logger # noqa: ABS101
|
||||
|
||||
logger = get_named_logger("Models")
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowBaseModel:
|
||||
"""Common things for stuff in the model."""
|
||||
|
||||
def get(
|
||||
self,
|
||||
field_name: str,
|
||||
title: bool = True,
|
||||
**kwargs
|
||||
):
|
||||
"""Get reportable field tuple."""
|
||||
field_info = self.__dataclass_fields__.get(field_name)
|
||||
# provide an empty string default title to minimise drama
|
||||
field_title = field_info.metadata.get("title", "")
|
||||
value = self.get_reportable_value(field_name=field_name, **kwargs)
|
||||
if title:
|
||||
return (field_title, value)
|
||||
return value
|
||||
|
||||
def get_reportable_value(
|
||||
self,
|
||||
field_name: str,
|
||||
*,
|
||||
decimal_places: int = None,
|
||||
default_value: str = "N/A") -> Optional[str]:
|
||||
"""Get the value of a value and make it reportable."""
|
||||
# Get the field info using the field name
|
||||
field_info = self.__dataclass_fields__.get(field_name)
|
||||
if field_info is None:
|
||||
raise AttributeError(
|
||||
f"{field_name!r} is not a field on {self.__class__.__name__}"
|
||||
)
|
||||
|
||||
value = getattr(self, field_name)
|
||||
|
||||
if value is None:
|
||||
return default_value
|
||||
|
||||
if isinstance(value, (int, float)):
|
||||
if decimal_places:
|
||||
value = round(value, decimal_places)
|
||||
if value < 0.0001 or value > 99999999:
|
||||
value = f"{value:.2E}"
|
||||
else:
|
||||
if decimal_places:
|
||||
raise TypeError(
|
||||
"decimal_places is not a supported argument for a non-numeric.")
|
||||
|
||||
unit = field_info.metadata.get('unit')
|
||||
|
||||
if unit:
|
||||
return f"{value} {unit}"
|
||||
|
||||
return str(value)
|
||||
|
||||
|
||||
class SampleType(str, Enum):
|
||||
@ -86,10 +146,6 @@ class Sample:
|
||||
metadata={
|
||||
"title": "Sample alias",
|
||||
"description": "The alias for the sample given by the user"})
|
||||
barcode: str = field(
|
||||
metadata={
|
||||
"title": "Sample barcode",
|
||||
"description": "The physical barcode assigned to the sample"})
|
||||
sample_type: SampleType = field(
|
||||
metadata={
|
||||
"title": "Sample type",
|
||||
@ -98,10 +154,15 @@ class Sample:
|
||||
metadata={
|
||||
"title": "Sample pass",
|
||||
"description": "If true the sample has passed workflow checks"})
|
||||
barcode: str | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"title": "Sample barcode",
|
||||
"description": "The physical barcode assigned to the sample"})
|
||||
additional_identifiers: List[SampleIdentifier] = field(
|
||||
default_factory=list, metadata={
|
||||
"title": "Additional sample identifiers",
|
||||
"description": "Addition identifiers for the sample"})
|
||||
"description": "Additional identifiers for the sample"})
|
||||
sample_checks: list[CheckResult] = field(
|
||||
default_factory=list, metadata={
|
||||
"title": "Sample checks",
|
||||
@ -123,9 +184,9 @@ class Sample:
|
||||
|
||||
def get_sample_identifier(self, sample_identifier):
|
||||
"""Get a sample identifier given the identifier name."""
|
||||
for indentifier in self.additional_identifiers:
|
||||
if indentifier.name == sample_identifier:
|
||||
return indentifier.value
|
||||
for identifier in self.additional_identifiers:
|
||||
if identifier.name == sample_identifier:
|
||||
return identifier.value
|
||||
raise KeyError("Sample identifier not found")
|
||||
|
||||
def set_sample_identifier(self, name, value):
|
||||
@ -139,7 +200,42 @@ class Sample:
|
||||
def to_json(self, filename):
|
||||
"""Save class as JSON."""
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder)
|
||||
json.dump(asdict(self), f, default=str, indent=2)
|
||||
|
||||
def get_reportable_qc_status(self, max_criteria=4):
|
||||
"""Store global status of the sample and list of QC criteria to show.
|
||||
|
||||
:params max_criteria: Maximum number of criteria to be reported.
|
||||
"""
|
||||
# Store global status: pass/ failed
|
||||
qc_global_status = {"status": self.sample_pass, "scope": "QC status"}
|
||||
qc_criteria = []
|
||||
if self.sample_pass:
|
||||
qc_criteria.append(
|
||||
{"status": self.sample_pass, "scope": "All acceptance criteria met"}
|
||||
)
|
||||
else:
|
||||
# Report failed criteria until a maximum value
|
||||
for qc in self.sample_checks:
|
||||
if not qc.check_pass: # append criteria if failed
|
||||
qc_criteria.append(
|
||||
{
|
||||
"status": qc.check_pass,
|
||||
"category": qc.friendly_check_category(),
|
||||
"scope": qc.friendly_check_name(),
|
||||
}
|
||||
)
|
||||
if len(qc_criteria) > max_criteria:
|
||||
# Replace all the failed criteria, with a sentence with the number
|
||||
# instead of listing all of them.
|
||||
# Set status to False as more than max_criteria are failed.
|
||||
qc_criteria = [
|
||||
{
|
||||
"status": False,
|
||||
"scope": f"{len(qc_criteria)} acceptance criteria",
|
||||
},
|
||||
]
|
||||
return qc_global_status, qc_criteria
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -161,7 +257,7 @@ class RunStats:
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowResult():
|
||||
class WorkflowResult(WorkflowBaseModel):
|
||||
"""
|
||||
Definition for results that will be returned by this workflow.
|
||||
|
||||
@ -189,18 +285,73 @@ class WorkflowResult():
|
||||
default_factory=dict, metadata={
|
||||
"title": "Client fields",
|
||||
"description": "Arbitrary key-value pairs provided by the client"})
|
||||
versions: dict[str, Any] | None = field(
|
||||
default_factory=dict, metadata={
|
||||
"title": "Analysis tool versions",
|
||||
"description": """Key-value pairs collecting the
|
||||
software used and the corresponding versions"""})
|
||||
params: dict[str, Any] | None = field(
|
||||
default_factory=dict, metadata={
|
||||
"title": "Pertinent parameters",
|
||||
"description": """Key-value pairs with the
|
||||
options chosen by the user"""})
|
||||
|
||||
def load_client_fields(self, filename):
|
||||
"""Load client fields."""
|
||||
with open(filename) as f:
|
||||
try:
|
||||
client_fields = json.loads(f.read())
|
||||
# convert any lists into strings for display
|
||||
for key, value in client_fields.items():
|
||||
if isinstance(value, list):
|
||||
client_fields[key] = ', '.join(value)
|
||||
except json.decoder.JSONDecodeError:
|
||||
client_fields = {"error": "Error parsing client fields file."}
|
||||
|
||||
self.client_fields = client_fields
|
||||
return self.client_fields
|
||||
|
||||
def load_params(self, params_json, keep=None):
|
||||
"""Create a workflow params dict."""
|
||||
params_json = Path(params_json)
|
||||
if keep is None:
|
||||
keep = []
|
||||
if not params_json.is_file():
|
||||
raise FileNotFoundError(f"No such file: {params_json}")
|
||||
with open(params_json, "r") as f:
|
||||
try:
|
||||
params_dict = json.loads(f.read())
|
||||
self.params = {
|
||||
k: v for k, v in params_dict.items() if k in set(keep)
|
||||
}
|
||||
return self.params
|
||||
except ValueError:
|
||||
raise ValueError(f"Invalid JSON file: {params_json}")
|
||||
|
||||
def load_versions(self, versions_path):
|
||||
"""Create a version list of dict."""
|
||||
versions_path = Path(versions_path)
|
||||
if not versions_path.exists():
|
||||
raise FileNotFoundError(f"No such file: {versions_path}")
|
||||
|
||||
if versions_path.is_dir():
|
||||
version_files = [
|
||||
vp for vp in versions_path.iterdir() if vp.is_file()
|
||||
]
|
||||
elif versions_path.is_file():
|
||||
version_files = [versions_path]
|
||||
else:
|
||||
raise IOError(f"{versions_path} should be either a directory or a file")
|
||||
for fname in version_files:
|
||||
versions = {}
|
||||
with open(fname, "r", encoding="utf-8") as fh:
|
||||
for line in fh.readlines():
|
||||
name, version = line.strip().split(",")
|
||||
versions[name] = version
|
||||
self.versions = versions
|
||||
return self.versions
|
||||
|
||||
def to_json(self, filename):
|
||||
"""Save class as JSON."""
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(asdict(self), f, default=str, indent=2, cls=DecimalEncoder)
|
||||
|
||||
|
||||
class DecimalEncoder(json.JSONEncoder):
|
||||
"""This should probably be moved."""
|
||||
|
||||
def default(self, obj):
|
||||
"""Override the default method to handle Decimal objects."""
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
return super().default(obj)
|
||||
json.dump(asdict(self), f, default=str, indent=2)
|
||||
|
||||
@ -9,7 +9,7 @@ therefore Nextflow will need to be
|
||||
installed before attempting to run the workflow.
|
||||
|
||||
The workflow can currently be run using either
|
||||
[Docker](https://www.docker.com/products/docker-desktop)
|
||||
[Docker](https://docs.docker.com/get-started/)
|
||||
or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html)
|
||||
to provide isolation of the required software.
|
||||
Both methods are automated out-of-the-box provided
|
||||
|
||||
@ -326,14 +326,16 @@ class NfcoreTemplate {
|
||||
String workflow_version = version(workflow)
|
||||
String.format(
|
||||
"""
|
||||
${colors.igreen}|||||||||| ${colors.reset}${colors.dim}_____ ____ ___ ____ __ __ _____ _ _
|
||||
${colors.igreen}|||||||||| ${colors.reset}${colors.dim}| ____| _ \\_ _|___ \\| \\/ | ____| | | __ _| |__ ___
|
||||
${colors.yellow}||||| ${colors.reset}${colors.dim}| _| | |_) | | __) | |\\/| | _| _____| |/ _` | '_ \\/ __|
|
||||
${colors.yellow}||||| ${colors.reset}${colors.dim}| |___| __/| | / __/| | | | |__|_____| | (_| | |_) \\__ \\
|
||||
${colors.iblue}|||||||||| ${colors.reset}${colors.dim}|_____|_| |___|_____|_| |_|_____| |_|\\__,_|_.__/|___/
|
||||
${colors.igreen}|||||||||| ${colors.reset}${colors.dim}_____ ____ ___ ____ __ __ _____
|
||||
${colors.igreen}|||||||||| ${colors.reset}${colors.dim}| ____| _ \\_ _|___ \\| \\/ | ____|
|
||||
${colors.yellow}||||| ${colors.reset}${colors.dim}| _| | |_) | | __) | |\\/| | _|
|
||||
${colors.yellow}||||| ${colors.reset}${colors.dim}| |___| __/| | / __/| | | | |__
|
||||
${colors.iblue}|||||||||| ${colors.reset}${colors.dim}|_____|_| |___|_____|_| |_|_____|
|
||||
${colors.iblue}|||||||||| ${colors.reset}${colors.bold}${workflow_name} ${workflow_version}${colors.reset}
|
||||
${NfcoreTemplate.dashedLine(monochrome_logs)}
|
||||
""".stripIndent()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -48,13 +48,6 @@ class Pinguscript {
|
||||
String paramsJSON = new JsonBuilder(params).toPrettyString()
|
||||
def params_data = new JsonSlurper().parseText(paramsJSON)
|
||||
|
||||
// hostname
|
||||
def host = null
|
||||
try {
|
||||
host = InetAddress.getLocalHost().getHostName()
|
||||
}
|
||||
catch(Exception e) {}
|
||||
|
||||
// OS
|
||||
// TODO check version on WSL
|
||||
def opsys = System.properties['os.name'].toLowerCase()
|
||||
@ -103,7 +96,7 @@ class Pinguscript {
|
||||
body_json \
|
||||
"tracking_id": [
|
||||
"msg_id": UUID.randomUUID().toString(),
|
||||
"version": "3.0.0"
|
||||
"version": "3.0.1"
|
||||
],
|
||||
"source": "workflow",
|
||||
"event": event,
|
||||
@ -123,7 +116,6 @@ class Pinguscript {
|
||||
],
|
||||
"env": [
|
||||
"user": user, // placeholder for any future okta
|
||||
"hostname": host,
|
||||
"os": [
|
||||
"name": opsys,
|
||||
"version": opver
|
||||
|
||||
@ -96,7 +96,7 @@ params {
|
||||
]
|
||||
agent = null
|
||||
container_sha = "shac733d952a14257cf3c5c5d5d44c6aed84d5fe5a1"
|
||||
common_sha = "sha9ef2f4e4585c4ce6a604616e77185077551abf50"
|
||||
common_sha = "sha1c69fd30053aad5d516e9567b3944384325a0fee"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -323,7 +323,12 @@
|
||||
"disable_ping": {
|
||||
"type": "boolean",
|
||||
"default": false,
|
||||
"description": "Enable to prevent sending a workflow ping."
|
||||
"description": "Enable to prevent sending a workflow ping.",
|
||||
"overrides": {
|
||||
"epi2mecloud": {
|
||||
"hidden": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"version": {
|
||||
"type": "boolean",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user