diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87626df..e950f47 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: always_run: true pass_filenames: false additional_dependencies: - - epi2melabs==0.0.56 + - epi2melabs==0.0.57 - id: build_models name: build_models entry: datamodel-codegen --strict-nullable --base-class workflow_glue.results_schema_helpers.BaseModel --use-schema-description --disable-timestamp --input results_schema.yml --input-file-type openapi --output bin/workflow_glue/results_schema.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f559026..9ea7221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [v1.4.0] ## Added - `--igv` parameter (default: false) for outputting IGV config allowing visualisation of read alignments in the EPI2ME App. - If required for IGV, reference indexes are output in to a `igv_reference` directory diff --git a/README.md b/README.md index dc1ba28..c31271a 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ therefore Nextflow will need to be installed before attempting to run the workflow. The workflow can currently be run using either -[Docker](https://www.docker.com/products/docker-desktop +[Docker](https://www.docker.com/products/docker-desktop) or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html) to provide isolation of the required software. Both methods are automated out-of-the-box provided diff --git a/bin/workflow_glue/check_bam_headers_in_dir.py b/bin/workflow_glue/check_bam_headers_in_dir.py index 44e689b..199e056 100755 --- a/bin/workflow_glue/check_bam_headers_in_dir.py +++ b/bin/workflow_glue/check_bam_headers_in_dir.py @@ -29,7 +29,13 @@ def main(args): for xam_file in target_files: # get the `@SQ` and `@HD` lines in the header with pysam.AlignmentFile(xam_file, check_sq=False) as f: - sq_lines = f.header.get("SQ") + # compare only the SN/LN/M5 elements of SQ to avoid labelling XAM with + # same reference but different SQ.UR as mixed_header (see CW-4842) + sq_lines = [{ + "SN": sq["SN"], + "LN": sq["LN"], + "M5": sq.get("M5"), + } for sq in f.header.get("SQ", [])] hd_lines = f.header.get("HD") # Check if it is sorted. # When there is more than one BAM, merging/sorting diff --git a/bin/workflow_glue/check_xam_index.py b/bin/workflow_glue/check_xam_index.py index 3beae14..f9f631e 100755 --- a/bin/workflow_glue/check_xam_index.py +++ b/bin/workflow_glue/check_xam_index.py @@ -14,12 +14,12 @@ def validate_xam_index(xam_file): Invalid indexes will fail the call with a ValueError: ValueError: fetch called on bamfile without index """ - alignments = pysam.AlignmentFile(xam_file, check_sq=False) - try: - alignments.fetch() - has_valid_index = True - except ValueError: - has_valid_index = False + with pysam.AlignmentFile(xam_file, check_sq=False) as alignments: + try: + alignments.fetch() + has_valid_index = True + except ValueError: + has_valid_index = False return has_valid_index diff --git a/docs/04_install_and_run.md b/docs/04_install_and_run.md index 3530988..9dfc367 100644 --- a/docs/04_install_and_run.md +++ b/docs/04_install_and_run.md @@ -9,7 +9,7 @@ therefore Nextflow will need to be installed before attempting to run the workflow. The workflow can currently be run using either -[Docker](https://www.docker.com/products/docker-desktop +[Docker](https://www.docker.com/products/docker-desktop) or [Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html) to provide isolation of the required software. Both methods are automated out-of-the-box provided diff --git a/lib/common.nf b/lib/common.nf index 2a31d49..3a8568d 100644 --- a/lib/common.nf +++ b/lib/common.nf @@ -15,6 +15,7 @@ process getParams { } process configure_igv { + publishDir "${params.out_dir}/", mode: 'copy', pattern: 'igv.json', enabled: params.containsKey("igv") && params.igv label "wf_common" cpus 1 memory "2 GB" diff --git a/lib/ingress.nf b/lib/ingress.nf index 6d14a83..2931357 100644 --- a/lib/ingress.nf +++ b/lib/ingress.nf @@ -197,15 +197,15 @@ def fastq_ingress(Map arguments) .map { meta, files, stats -> // new `arity: '1..*'` would be nice here files = files instanceof List ? files : [files] - new_keys = [ + def new_keys = [ "group_key": groupKey(meta["alias"], files.size()), "n_fastq": files.size()] - grp_index = (0.. - new_keys = [ + def new_keys = [ "group_index": "${meta["alias"]}_${grp_i}"] [meta + new_keys, files, stats] } @@ -279,17 +279,19 @@ def xam_ingress(Map arguments) // sorted, the index will be used. meta, paths -> boolean is_array = paths instanceof ArrayList - String xai_fn + String src_xam + String src_xai // Using `.uri` or `.Uri()` leads to S3 paths to be prefixed with `s3:///` // instead of `s3://`, causing the workflow to not find the index file. // `.toUriString()` returns the correct path. if (!is_array){ + src_xam = paths.toUriString() def xai = file(paths.toUriString() + ".bai") if (xai.exists()){ - xai_fn = xai.toUriString() + src_xai = xai.toUriString() } } - [meta + [xai_fn: xai_fn], paths] + [meta + [src_xam: src_xam, src_xai: src_xai], paths] } | checkBamHeaders | map { meta, paths, is_unaligned_env, mixed_headers_env, is_sorted_env -> @@ -331,9 +333,9 @@ def xam_ingress(Map arguments) // - between 1 and `N_OPEN_FILES_LIMIT` aligned files no_files: n_files == 0 indexed: \ - n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && meta["xai_fn"] - to_index: - n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && !meta["xai_fn"] + n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && meta["src_xai"] + to_index: \ + n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && !meta["src_xai"] to_catsort: \ (n_files == 1) || (n_files > N_OPEN_FILES_LIMIT) || meta["is_unaligned"] to_merge: true @@ -358,20 +360,20 @@ def xam_ingress(Map arguments) .map { meta, files, stats -> // new `arity: '1..*'` would be nice here files = files instanceof List ? files : [files] - new_keys = [ + def new_keys = [ "group_key": groupKey(meta["alias"], files.size()), "n_fastq": files.size()] - grp_index = (0.. - new_keys = [ + def new_keys = [ "group_index": "${meta["alias"]}_${grp_i}"] [meta + new_keys, files, stats] } .map { meta, path, stats -> - [meta.findAll { it.key !in ['xai_fn', 'is_sorted'] }, path, stats] + [meta.findAll { it.key !in ['is_sorted', 'src_xam', 'src_xai'] }, path, stats] } // add number of reads, run IDs, and basecall models to meta @@ -388,10 +390,18 @@ def xam_ingress(Map arguments) | sortBam | groupTuple | mergeBams + | map{ + meta, bam, bai -> + [meta + [src_xam: null, src_xai: null], bam, bai] + } // now handle samples with too many files for `samtools merge` ch_catsorted = ch_result.to_catsort | catSortBams + | map{ + meta, bam, bai -> + [meta + [src_xam: null, src_xai: null], bam, bai] + } // Validate the index of the input BAM. // If the input BAM index is invalid, regenerate it. @@ -399,7 +409,7 @@ def xam_ingress(Map arguments) ch_to_validate = ch_result.indexed | map{ meta, paths -> - bai = paths && meta.xai_fn ? file(meta.xai_fn) : null + def bai = paths && meta.src_xai ? file(meta.src_xai) : null [meta, paths, bai] } | branch { @@ -429,6 +439,10 @@ def xam_ingress(Map arguments) ch_indexed = ch_result.to_index | mix( ch_validated.invalid_idx ) | samtools_index + | map{ + meta, bam, bai -> + [meta + [src_xai: null], bam, bai] + } // Add extra null for the missing index to input.missing // as well as the missing metadata. @@ -439,7 +453,7 @@ def xam_ingress(Map arguments) ) | map{ meta, paths -> - [meta + [xai_fn: null, is_sorted: false], paths, null] + [meta + [src_xam: null, src_xai: null, is_sorted: false], paths, null] } // Combine all possible inputs @@ -480,7 +494,7 @@ def xam_ingress(Map arguments) } // Remove metadata that are unnecessary downstream: - // meta.xai_fn: not needed, as it will be part of the channel as a file + // meta.src_xai: not needed, as it will be part of the channel as a file // meta.is_sorted: if data are aligned, they will also be sorted/indexed // // The output meta can contain the following flags: @@ -498,7 +512,7 @@ def xam_ingress(Map arguments) ch_result | map{ meta, bam, bai, stats -> - [meta.findAll { it.key !in ['xai_fn', 'is_sorted'] }, [bam, bai], stats] + [meta.findAll { it.key !in ['is_sorted'] }, [bam, bai], stats] }, "xam" ) @@ -508,6 +522,19 @@ def xam_ingress(Map arguments) | map{ it.flatten() } + // Final check to ensure that src_xam/src_xai is not an s3 + // path. If so, drop it. We check src_xam also for src_xai + // as, the latter is irrelevant if the former is in s3. + | map{ + meta, bam, bai, stats -> + def xam = meta.src_xam + def xai = meta.src_xai + if (meta.src_xam){ + xam = meta.src_xam.startsWith('s3://') ? null : meta.src_xam + xai = meta.src_xam.startsWith('s3://') ? null : meta.src_xai + } + [ meta + [src_xam: xam, src_xai: xai], bam, bai, stats ] + } return ch_result } diff --git a/nextflow.config b/nextflow.config index ebde97e..136cacd 100644 --- a/nextflow.config +++ b/nextflow.config @@ -96,7 +96,7 @@ params { ] agent = null container_sha = "shafb1e2372e1535f0b42891ed2c68ffdac2ca1d658" - common_sha = "shad399cf22079b5b153920ac39ee40095a677933f1" + common_sha = "shae58638742cf84dbeeec683ba24bcdee67f64b986" } } @@ -107,7 +107,7 @@ manifest { description = 'Transcriptome analysis including differential expression as well as assembly and annotation of cDNA and direct RNA sequencing data.' mainScript = 'main.nf' nextflowVersion = '>=23.04.2' - version = 'v1.3.0' + version = 'v1.4.0' } epi2melabs {