kfukuta/wf-transcriptomes-v202
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'template-updates' into 'dev'

Browse Source 
Template updates

See merge request epi2melabs/workflows/wf-transcriptomes!183
This commit is contained in:
Sarah Griffiths 2024-09-11 12:47:01 +00:00
commit ce4bfa9a30
9 changed files with 64 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.pre-commit-config.yaml
View File

@ -8,7 +8,7 @@ repos:
always_run: true
pass_filenames: false
additional_dependencies:
- epi2melabs==0.0.56
- epi2melabs==0.0.57
- id: build_models
name: build_models
entry: datamodel-codegen --strict-nullable --base-class workflow_glue.results_schema_helpers.BaseModel --use-schema-description --disable-timestamp --input results_schema.yml --input-file-type openapi --output bin/workflow_glue/results_schema.py

2
CHANGELOG.md
View File

@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [unreleased]
## [v1.4.0]
## Added
- `--igv` parameter (default: false) for outputting IGV config allowing visualisation of read alignments in the EPI2ME App.
- If required for IGV, reference indexes are output in to a `igv_reference` directory

2
README.md
View File

@ -48,7 +48,7 @@ therefore Nextflow will need to be
installed before attempting to run the workflow.
The workflow can currently be run using either
[Docker](https://www.docker.com/products/docker-desktop
[Docker](https://www.docker.com/products/docker-desktop)
or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html)
to provide isolation of the required software.
Both methods are automated out-of-the-box provided

8
bin/workflow_glue/check_bam_headers_in_dir.py
View File

@ -29,7 +29,13 @@ def main(args):
for xam_file in target_files:
# get the `@SQ` and `@HD` lines in the header
with pysam.AlignmentFile(xam_file, check_sq=False) as f:
sq_lines = f.header.get("SQ")
# compare only the SN/LN/M5 elements of SQ to avoid labelling XAM with
# same reference but different SQ.UR as mixed_header (see CW-4842)
sq_lines = [{
"SN": sq["SN"],
"LN": sq["LN"],
"M5": sq.get("M5"),
} for sq in f.header.get("SQ", [])]
hd_lines = f.header.get("HD")
# Check if it is sorted.
# When there is more than one BAM, merging/sorting

12
bin/workflow_glue/check_xam_index.py
View File

@ -14,12 +14,12 @@ def validate_xam_index(xam_file):
Invalid indexes will fail the call with a ValueError:
ValueError: fetch called on bamfile without index
"""
alignments = pysam.AlignmentFile(xam_file, check_sq=False)
try:
alignments.fetch()
has_valid_index = True
except ValueError:
has_valid_index = False
with pysam.AlignmentFile(xam_file, check_sq=False) as alignments:
try:
alignments.fetch()
has_valid_index = True
except ValueError:
has_valid_index = False
return has_valid_index

2
docs/04_install_and_run.md
View File

@ -9,7 +9,7 @@ therefore Nextflow will need to be
installed before attempting to run the workflow.
The workflow can currently be run using either
[Docker](https://www.docker.com/products/docker-desktop
[Docker](https://www.docker.com/products/docker-desktop)
or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html)
to provide isolation of the required software.
Both methods are automated out-of-the-box provided

1
lib/common.nf
View File

@ -15,6 +15,7 @@ process getParams {
}
process configure_igv {
publishDir "${params.out_dir}/", mode: 'copy', pattern: 'igv.json', enabled: params.containsKey("igv") && params.igv
label "wf_common"
cpus 1
memory "2 GB"

61
lib/ingress.nf
View File

@ -197,15 +197,15 @@ def fastq_ingress(Map arguments)
.map { meta, files, stats ->
// new `arity: '1..*'` would be nice here
files = files instanceof List ? files : [files]
new_keys = [
def new_keys = [
"group_key": groupKey(meta["alias"], files.size()),
"n_fastq": files.size()]
grp_index = (0..<files.size()).collect()
def grp_index = (0..<files.size()).collect()
[meta + new_keys, files, grp_index, stats]
}
.transpose(by: [1, 2]) // spread multiple fastq files into separate emissions
.map { meta, files, grp_i, stats ->
new_keys = [
def new_keys = [
"group_index": "${meta["alias"]}_${grp_i}"]
[meta + new_keys, files, stats]
}
@ -279,17 +279,19 @@ def xam_ingress(Map arguments)
// sorted, the index will be used.
meta, paths ->
boolean is_array = paths instanceof ArrayList
String xai_fn
String src_xam
String src_xai
// Using `.uri` or `.Uri()` leads to S3 paths to be prefixed with `s3:///`
// instead of `s3://`, causing the workflow to not find the index file.
// `.toUriString()` returns the correct path.
if (!is_array){
src_xam = paths.toUriString()
def xai = file(paths.toUriString() + ".bai")
if (xai.exists()){
xai_fn = xai.toUriString()
src_xai = xai.toUriString()
}
}
[meta + [xai_fn: xai_fn], paths]
[meta + [src_xam: src_xam, src_xai: src_xai], paths]
}
| checkBamHeaders
| map { meta, paths, is_unaligned_env, mixed_headers_env, is_sorted_env ->
@ -331,9 +333,9 @@ def xam_ingress(Map arguments)
// - between 1 and `N_OPEN_FILES_LIMIT` aligned files
no_files: n_files == 0
indexed: \
n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && meta["xai_fn"]
to_index:
n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && !meta["xai_fn"]
n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && meta["src_xai"]
to_index: \
n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && !meta["src_xai"]
to_catsort: \
(n_files == 1) || (n_files > N_OPEN_FILES_LIMIT) || meta["is_unaligned"]
to_merge: true
@ -358,20 +360,20 @@ def xam_ingress(Map arguments)
.map { meta, files, stats ->
// new `arity: '1..*'` would be nice here
files = files instanceof List ? files : [files]
new_keys = [
def new_keys = [
"group_key": groupKey(meta["alias"], files.size()),
"n_fastq": files.size()]
grp_index = (0..<files.size()).collect()
def grp_index = (0..<files.size()).collect()
[meta + new_keys, files, grp_index, stats]
}
.transpose(by: [1, 2]) // spread multiple fastq files into separate emissions
.map { meta, files, grp_i, stats ->
new_keys = [
def new_keys = [
"group_index": "${meta["alias"]}_${grp_i}"]
[meta + new_keys, files, stats]
}
.map { meta, path, stats ->
[meta.findAll { it.key !in ['xai_fn', 'is_sorted'] }, path, stats]
[meta.findAll { it.key !in ['is_sorted', 'src_xam', 'src_xai'] }, path, stats]
}
// add number of reads, run IDs, and basecall models to meta
@ -388,10 +390,18 @@ def xam_ingress(Map arguments)
| sortBam
| groupTuple
| mergeBams
| map{
meta, bam, bai ->
[meta + [src_xam: null, src_xai: null], bam, bai]
}
// now handle samples with too many files for `samtools merge`
ch_catsorted = ch_result.to_catsort
| catSortBams
| map{
meta, bam, bai ->
[meta + [src_xam: null, src_xai: null], bam, bai]
}
// Validate the index of the input BAM.
// If the input BAM index is invalid, regenerate it.
@ -399,7 +409,7 @@ def xam_ingress(Map arguments)
ch_to_validate = ch_result.indexed
| map{
meta, paths ->
bai = paths && meta.xai_fn ? file(meta.xai_fn) : null
def bai = paths && meta.src_xai ? file(meta.src_xai) : null
[meta, paths, bai]
}
| branch {
@ -429,6 +439,10 @@ def xam_ingress(Map arguments)
ch_indexed = ch_result.to_index
| mix( ch_validated.invalid_idx )
| samtools_index
| map{
meta, bam, bai ->
[meta + [src_xai: null], bam, bai]
}
// Add extra null for the missing index to input.missing
// as well as the missing metadata.
@ -439,7 +453,7 @@ def xam_ingress(Map arguments)
)
| map{
meta, paths ->
[meta + [xai_fn: null, is_sorted: false], paths, null]
[meta + [src_xam: null, src_xai: null, is_sorted: false], paths, null]
}
// Combine all possible inputs
@ -480,7 +494,7 @@ def xam_ingress(Map arguments)
}
// Remove metadata that are unnecessary downstream:
// meta.xai_fn: not needed, as it will be part of the channel as a file
// meta.src_xai: not needed, as it will be part of the channel as a file
// meta.is_sorted: if data are aligned, they will also be sorted/indexed
//
// The output meta can contain the following flags:
@ -498,7 +512,7 @@ def xam_ingress(Map arguments)
ch_result
| map{
meta, bam, bai, stats ->
[meta.findAll { it.key !in ['xai_fn', 'is_sorted'] }, [bam, bai], stats]
[meta.findAll { it.key !in ['is_sorted'] }, [bam, bai], stats]
},
"xam"
)
@ -508,6 +522,19 @@ def xam_ingress(Map arguments)
| map{
it.flatten()
}
// Final check to ensure that src_xam/src_xai is not an s3
// path. If so, drop it. We check src_xam also for src_xai
// as, the latter is irrelevant if the former is in s3.
| map{
meta, bam, bai, stats ->
def xam = meta.src_xam
def xai = meta.src_xai
if (meta.src_xam){
xam = meta.src_xam.startsWith('s3://') ? null : meta.src_xam
xai = meta.src_xam.startsWith('s3://') ? null : meta.src_xai
}
[ meta + [src_xam: xam, src_xai: xai], bam, bai, stats ]
}
return ch_result
}

4
nextflow.config
View File

@ -96,7 +96,7 @@ params {
]
agent = null
container_sha = "shafb1e2372e1535f0b42891ed2c68ffdac2ca1d658"
common_sha = "shad399cf22079b5b153920ac39ee40095a677933f1"
common_sha = "shae58638742cf84dbeeec683ba24bcdee67f64b986"
}
}
@ -107,7 +107,7 @@ manifest {
description = 'Transcriptome analysis including differential expression as well as assembly and annotation of cDNA and direct RNA sequencing data.'
mainScript = 'main.nf'
nextflowVersion = '>=23.04.2'
version = 'v1.3.0'
version = 'v1.4.0'
}
epi2melabs {