From 14fe76dabc464bd55a57f82faa2f9c6e2a5315e0 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Fri, 6 Sep 2024 09:27:56 +0000 Subject: [PATCH] CW-4613 IGV config --- .gitlab-ci.yml | 55 +++++++++++---- CHANGELOG.md | 7 ++ README.md | 11 +++ docs/06_input_parameters.md | 1 + docs/07_outputs.md | 5 ++ docs/08_pipeline_overview.md | 2 + docs/09_troubleshooting.md | 3 +- docs/10_FAQ.md | 2 + main.nf | 130 ++++++++++++++++++++++++++++++++++- nextflow.config | 3 +- nextflow_schema.json | 7 ++ output_definition.json | 40 +++++++++++ 12 files changed, 251 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa3e260..6322847 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,7 +53,7 @@ docker-run: "only_differential_expression", "differential_expression_gff3", "ncbi_gzip", "ncbi_no_gene_id", "ensembl_with_versions", "differential_expression_mouse", "no_ref_annotation", - "unstranded_annotation_error" + "unstranded_annotation_error", "igv", "igv_fai_gz" ] rules: # NOTE As we're overriding the rules block for the included docker-run @@ -67,21 +67,21 @@ docker-run: NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" - NF_IGNORE_PROCESSES: preprocess_reads,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "isoforms_bam" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome-source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" - NF_IGNORE_PROCESSES: preprocess_reads,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "no_ref_annotation" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" - NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config @@ -91,7 +91,7 @@ docker-run: --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" - NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "only_differential_expression" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config @@ -105,7 +105,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_gff3" variables: @@ -120,7 +120,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ncbi_gzip" variables: @@ -135,7 +135,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts, + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ncbi_no_gene_id" variables: @@ -149,7 +149,7 @@ docker-run: --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts, + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ensembl_with_versions" variables: @@ -163,7 +163,7 @@ docker-run: --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts, + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_mouse" variables: @@ -177,7 +177,7 @@ docker-run: --transcriptome_assembly false --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression_mouse/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation, + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam - if: $MATRIX_NAME == "unstranded_annotation_error" variables: @@ -188,6 +188,37 @@ docker-run: --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/unstranded_annotation.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" - NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome ASSERT_NEXTFLOW_FAILURE: "test_fail" # set to any non-zero length str to allow the nextflow CMD to fail ASSERT_NEXTFLOW_FAILURE_REXP: "In ref_annotation, transcript features must have a strand of either '+' or '-'" + - if: $MATRIX_NAME == "igv" + variables: + NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config + NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ + --de_analysis \ + --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ + --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \ + --direct_rna --minimap2_index_opts '-k 15' \ + --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \ + --sample_sheet test_data/sample_sheet.csv \ + --igv \ + -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" + NF_IGNORE_PROCESSES: > + preprocess_reads,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, + build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + - if: $MATRIX_NAME == "igv_fai_gz" + variables: + NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config + NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ + --fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \ + --transcriptome-source precomputed --de_analysis \ + --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \ + --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \ + --direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \ + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \ + --igv \ + -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" + NF_IGNORE_PROCESSES: > + preprocess_reads,merge_transcriptomes,assemble_transcripts, + build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + diff --git a/CHANGELOG.md b/CHANGELOG.md index 4259fee..f559026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +## Added +- `--igv` parameter (default: false) for outputting IGV config allowing visualisation of read alignments in the EPI2ME App. +- If required for IGV, reference indexes are output in to a `igv_reference` directory +### Changed +- BAMS are output in to a BAMS directory. + ## [v1.3.0] ### Removed - Fusion detection subworkflow, as the functionality is not robust enough for general use at this time. diff --git a/README.md b/README.md index e77b6b3..dc1ba28 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ input_reads.fastq ─── input_directory ─── input_directory | Nextflow parameter name | Type | Description | Help | Default | |--------------------------|------|-------------|------|---------| | out_dir | string | Directory for output of all user-facing files. | | output | +| igv | boolean | Visualize outputs in the EPI2ME IGV visualizer. | Enabling this option will visualize the output alignment files in the EPI2ME Desktop App IGV visualizer. | False | ### Sample Options @@ -232,6 +233,11 @@ Output files may be aggregated including information for all samples or provided | Transcript counts filtered | de_analysis/filtered_transcript_counts_with_genes.tsv | Filtered transcript counts, used for differential transcript usage analysis. Includes a reference to the associated gene ID. | aggregated | | Transcript info table | {{ alias }}_transcripts_table.tsv | This file details each isoform that was reconstructed from the input reads. It contains a subset of columns from the .tmap output from [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) | per-sample | | Final non redundant transcriptome | de_analysis/final_non_redundant_transcriptome.fasta | Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. | aggregated | +| Index of reference FASTA file | igv_reference/{{ ref_genome file }}.fai | Reference genome index of the FASTA file required for IGV config. | aggregated | +| GZI index of the reference FASTA file | igv_reference/{{ ref_genome file }}.gzi | GZI Index of the reference FASTA file. | aggregated | +| JSON configuration file for IGV browser | igv.json | JSON configuration file to be loaded in IGV for visualising alignments against the reference. | aggregated | +| BAM file (minimap2) | BAMS/{{ alias }}.reads_aln_sorted.bam | BAM file generated from mapping input reads to the reference. | per-sample | +| BAM index file (minimap2) | BAMS/{{ alias }}.reads_aln_sort.bam.bai | Index file generated from mapping input reads to the reference. | per-sample | @@ -250,6 +256,8 @@ If the `transcriptome_source` parameter is "reference-guided" a transcriptome wi #### 3.1 Align reads with reference genome. The reference genome will be indexed and aligned using [Minimap2](https://github.com/lh3/minimap2). The output is sorted and converted to a BAM file using [Samtools](https://www.htslib.org/). Alignment stats are created from these using [Seqkit BAM](https://bioinf.shenwei.me/seqkit/usage/#bam). +Additionally, the workflow will generate an IGV configuration file if `--igv` is selected. This file allows the user to view the aligned BAM in the EPI2ME Desktop Application in the Viewer tab. + #### 3.2 Chunk BAM The aligned BAMs are split into chunks using the bundle_min_reads parameter (default: 50000). @@ -316,6 +324,7 @@ The final component of this isoform analysis is a stage-wise statistical test us + If the workflow fails please run it with the demo data set to ensure the workflow itself is working. This will help us determine if the issue is related to the environment, input parameters or a bug. + See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/). ++ Renaming, moving or deleting the input BAM, reference genome or the output directory from the location provided at runtime will stop IGV in the EPI2ME Desktop app from loading. @@ -323,6 +332,8 @@ The final component of this isoform analysis is a stage-wise statistical test us *Does the workflow support de novo assembly?* - Currently the workflow does not have a *de novo* mode. +*Why is the IGV panel not showing?* - The workflow expects either an uncompressed or [`bgzip`](https://www.htslib.org/doc/bgzip.html)-compressed reference. If the user provides a reference compressed not with `bgzip`, the workflow will run to completion, but won't be able to generate the necessary indexes to visualize the outputs in IGV. + If your question is not answered here, please report any issues or suggestions on the [github issues](https://github.com/epi2me-labs/wf-transcriptomes/issues) page or start a discussion on the [community](https://community.nanoporetech.com/). diff --git a/docs/06_input_parameters.md b/docs/06_input_parameters.md index 5bff6f3..62ba1fc 100644 --- a/docs/06_input_parameters.md +++ b/docs/06_input_parameters.md @@ -17,6 +17,7 @@ | Nextflow parameter name | Type | Description | Help | Default | |--------------------------|------|-------------|------|---------| | out_dir | string | Directory for output of all user-facing files. | | output | +| igv | boolean | Visualize outputs in the EPI2ME IGV visualizer. | Enabling this option will visualize the output alignment files in the EPI2ME Desktop App IGV visualizer. | False | ### Sample Options diff --git a/docs/07_outputs.md b/docs/07_outputs.md index cc1260a..33ecfa0 100644 --- a/docs/07_outputs.md +++ b/docs/07_outputs.md @@ -26,3 +26,8 @@ Output files may be aggregated including information for all samples or provided | Transcript counts filtered | de_analysis/filtered_transcript_counts_with_genes.tsv | Filtered transcript counts, used for differential transcript usage analysis. Includes a reference to the associated gene ID. | aggregated | | Transcript info table | {{ alias }}_transcripts_table.tsv | This file details each isoform that was reconstructed from the input reads. It contains a subset of columns from the .tmap output from [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) | per-sample | | Final non redundant transcriptome | de_analysis/final_non_redundant_transcriptome.fasta | Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. | aggregated | +| Index of reference FASTA file | igv_reference/{{ ref_genome file }}.fai | Reference genome index of the FASTA file required for IGV config. | aggregated | +| GZI index of the reference FASTA file | igv_reference/{{ ref_genome file }}.gzi | GZI Index of the reference FASTA file. | aggregated | +| JSON configuration file for IGV browser | igv.json | JSON configuration file to be loaded in IGV for visualising alignments against the reference. | aggregated | +| BAM file (minimap2) | BAMS/{{ alias }}.reads_aln_sorted.bam | BAM file generated from mapping input reads to the reference. | per-sample | +| BAM index file (minimap2) | BAMS/{{ alias }}.reads_aln_sort.bam.bai | Index file generated from mapping input reads to the reference. | per-sample | diff --git a/docs/08_pipeline_overview.md b/docs/08_pipeline_overview.md index 0fa5b66..cd38089 100644 --- a/docs/08_pipeline_overview.md +++ b/docs/08_pipeline_overview.md @@ -10,6 +10,8 @@ If the `transcriptome_source` parameter is "reference-guided" a transcriptome wi #### 3.1 Align reads with reference genome. The reference genome will be indexed and aligned using [Minimap2](https://github.com/lh3/minimap2). The output is sorted and converted to a BAM file using [Samtools](https://www.htslib.org/). Alignment stats are created from these using [Seqkit BAM](https://bioinf.shenwei.me/seqkit/usage/#bam). +Additionally, the workflow will generate an IGV configuration file if `--igv` is selected. This file allows the user to view the aligned BAM in the EPI2ME Desktop Application in the Viewer tab. + #### 3.2 Chunk BAM The aligned BAMs are split into chunks using the bundle_min_reads parameter (default: 50000). diff --git a/docs/09_troubleshooting.md b/docs/09_troubleshooting.md index ecb81f2..d5ee7d1 100644 --- a/docs/09_troubleshooting.md +++ b/docs/09_troubleshooting.md @@ -1,2 +1,3 @@ + If the workflow fails please run it with the demo data set to ensure the workflow itself is working. This will help us determine if the issue is related to the environment, input parameters or a bug. -+ See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/). \ No newline at end of file ++ See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/). ++ Renaming, moving or deleting the input BAM, reference genome or the output directory from the location provided at runtime will stop IGV in the EPI2ME Desktop app from loading. \ No newline at end of file diff --git a/docs/10_FAQ.md b/docs/10_FAQ.md index 7d65676..ad4d36c 100644 --- a/docs/10_FAQ.md +++ b/docs/10_FAQ.md @@ -1,3 +1,5 @@ *Does the workflow support de novo assembly?* - Currently the workflow does not have a *de novo* mode. +*Why is the IGV panel not showing?* - The workflow expects either an uncompressed or [`bgzip`](https://www.htslib.org/doc/bgzip.html)-compressed reference. If the user provides a reference compressed not with `bgzip`, the workflow will run to completion, but won't be able to generate the necessary indexes to visualize the outputs in IGV. + If your question is not answered here, please report any issues or suggestions on the [github issues](https://github.com/epi2me-labs/wf-transcriptomes/issues) page or start a discussion on the [community](https://community.nanoporetech.com/). \ No newline at end of file diff --git a/main.nf b/main.nf index c2d12fe..c6d8330 100644 --- a/main.nf +++ b/main.nf @@ -10,6 +10,7 @@ import java.util.ArrayList; nextflow.enable.dsl = 2 include { fastq_ingress; xam_ingress } from './lib/ingress' +include { configure_igv } from './lib/common' include { reference_assembly } from './subworkflows/reference_assembly' include { differential_expression } from './subworkflows/differential_expression' @@ -526,6 +527,50 @@ process check_annotation_strand { } +// Process to create the faidx index +process faidx { + // If the input file is gzipped, we need to emit the indexes for the input gzip file + // only. Therefore, this become redundant to be emitted as it won't be used by the + // IGV configuration, but only by internal processes together with the decompressed + // FASTA file. To avoid unnecessary emissions, we enable only if the input file is + // decompressed. + publishDir "${params.out_dir}/igv_reference", mode: 'copy', pattern: "*", enabled: !params.ref_genome.toLowerCase().endsWith("gz") + label "wf_common" + cpus 1 + memory 4.GB + input: + path(ref) + output: + path("${ref}.fai") + script: + """ + samtools faidx ${ref} + """ +} + + +// Process to create the faidx indexes for a gzipped reference +process gz_faidx { + publishDir "${params.out_dir}/igv_reference", mode: 'copy', pattern: "*" + label "wf_common" + cpus 1 + memory 4.GB + // If a user provides a non-bgzipped file, the process won't + // generate the indexes. We should tolerate that, still avoid emitting + // the reference and simply have a broken IGV file. + // The gzi is not required to operate the workflow, so we actually tolerate any failure. + errorStrategy 'ignore' + input: + path(ref) + output: + tuple path("${ref}.fai"), path("${ref}.gzi") + script: + """ + samtools faidx ${ref} + """ +} + + // workflow module workflow pipeline { @@ -580,6 +625,8 @@ workflow pipeline { results = Channel.empty() + // Define BAM output Directory + String publish_bams = "BAMS" software_versions = getVersions() workflow_params = getParams() input_reads = reads.map{ meta, samples, stats -> [meta, samples]} @@ -634,7 +681,9 @@ workflow pipeline { merge_gff = merge_gff_bundles.out.gff.map{ it -> it[1]}.collect() - results = results.concat(assembly.bam.map {sample_id, bam, bai -> [bam, bai]}.flatten()) + // Output BAMS in a dedicated directory + bam_results = assembly.bam.map{ + sample_id, bam, bai -> [bam, bai]}.flatten().map{ [it, publish_bams] } } else{ gff_compare = OPTIONAL_FILE @@ -723,6 +772,85 @@ workflow pipeline { } results.concat(workflow_params.map{ [it, null]}) + // IGV config + if (params.transcriptome_source == "precomputed" && params.igv){ + log.warn("IGV configuration does not work if transcriptome sources is set to `precomputed`.") + } + if (params.transcriptome_source != "precomputed" && params.igv){ + is_compressed = file("${params.ref_genome}").extension == "gz" + String publish_ref = "igv_reference" + String current_dir = "${file('.').toUriString()}" + reference_genome = Channel.fromPath("${params.ref_genome}") + igv_ref = reference_genome | flatten | map { it -> "${it.toUriString()}" } + if (is_compressed){ + // Define indexes names. + String input_fai_index = "${params.ref_genome}.fai" + String input_gzi_index = "${params.ref_genome}.gzi" + // Check whether the input gzref is indexed. If so, pass these as indexes. + // Otherwise, generate the gzip + fai indexes for the compressed reference. + if (file(input_fai_index).exists() && file(input_gzi_index).exists()){ + gzindexes = Channel.fromPath(input_fai_index) + | mix( + Channel.fromPath(input_gzi_index) + ) + gz_igv = gzindexes | flatten | map { it -> "${it.toUriString()}" } + } else { + gz_igv = gz_faidx(Channel.fromPath("${params.ref_genome}")) + | flatten + | map { it -> "$current_dir/$params.out_dir/$publish_ref/${it.Name}" } + gz_igv | ifEmpty{ + if (params.containsKey("igv") && params.igv){ + log.warn """\ + The input reference is compressed but not with bgzip, which is required to create an index. + The workflow will proceed but it will not be possible to load the reference in the IGV Viewer. + To use the IGV Viewer, provide an uncompressed, or bgzip compressed version of the input reference next time you run the workflow. + """.stripIndent() + } + } + } + } else { + gzindexes = Channel.empty() + gz_igv = Channel.empty() + } + + // Generate fai index if the file is either compressed, or if fai doesn't exists + if (!is_compressed && file("${params.ref_genome}.fai").exists()){ + ref_idx = Channel.fromPath("${params.ref_genome}.fai") + igv_index = ref_idx | flatten | map { it -> "${it.toUriString()}" } + } else { + ref_idx = faidx(reference_genome) + igv_index = ref_idx | map { it -> "$current_dir/$params.out_dir/$publish_ref/${it.Name}" } + } + + // get list of file names + // Absolute paths required for directories + igv_files = reads + | map { meta, sample, stats -> meta.alias } + | toSortedList + | map { list -> list.collect{ + [ + "$current_dir/$params.out_dir/$publish_bams/${it}_reads_aln_sorted.bam", + "$current_dir/$params.out_dir/$publish_bams/${it}_reads_aln_sorted.bam.bai" + ] + } } + | concat ( igv_index) + | flatten + | concat (igv_ref) + | concat (gz_igv) + | flatten + | collectFile(name: "file-names.txt", newLine: true, sort: false) + + // configure IGV + igv_conf = configure_igv( + igv_files, + Channel.of(null), // igv locus + [displayMode: "SQUISHED", colorBy: "strand"], // bam extra opts + Channel.of(null), // vcf extra opts + ) + + results = results.concat(igv_conf.map{ [it, null]}) + results = results.concat(bam_results) + } emit: diff --git a/nextflow.config b/nextflow.config index e2af623..ebde97e 100644 --- a/nextflow.config +++ b/nextflow.config @@ -31,6 +31,7 @@ params { version = false monochrome_logs = false + igv = false validate_params = true show_hidden_params = false schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf' @@ -95,7 +96,7 @@ params { ] agent = null container_sha = "shafb1e2372e1535f0b42891ed2c68ffdac2ca1d658" - common_sha = "shab540ba556d0d8c38bea8fec520f0bdedd9e59520" + common_sha = "shad399cf22079b5b153920ac39ee40095a677933f1" } } diff --git a/nextflow_schema.json b/nextflow_schema.json index fae245f..c0982c1 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -110,6 +110,13 @@ "format": "directory-path", "default": "output", "description": "Directory for output of all user-facing files." + }, + "igv": { + "title": "Show in IGV", + "type": "boolean", + "default": false, + "description": "Visualize outputs in the EPI2ME IGV visualizer.", + "help_text": "Enabling this option will visualize the output alignment files in the EPI2ME Desktop App IGV visualizer." } } }, diff --git a/output_definition.json b/output_definition.json index 82811af..c223e4e 100644 --- a/output_definition.json +++ b/output_definition.json @@ -191,6 +191,46 @@ "mime-type": "text/x-fasta", "optional": true, "type": "aggregated" + }, + "reference-index": { + "filepath": "igv_reference/{{ ref_genome file }}.fai", + "title": "Index of reference FASTA file", + "description": "Reference genome index of the FASTA file required for IGV config.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "reference-gzi-index": { + "filepath": "igv_reference/{{ ref_genome file }}.gzi", + "title": "GZI index of the reference FASTA file", + "description": "GZI Index of the reference FASTA file.", + "mime-type": "application/octet-stream", + "optional": true, + "type": "aggregated" + }, + "igv-config": { + "filepath": "igv.json", + "title": "JSON configuration file for IGV browser", + "description": "JSON configuration file to be loaded in IGV for visualising alignments against the reference.", + "mime-type": "text/json", + "optional": true, + "type": "aggregated" + }, + "minimap2-bam": { + "filepath": "BAMS/{{ alias }}.reads_aln_sorted.bam", + "title": "BAM file (minimap2)", + "description": "BAM file generated from mapping input reads to the reference.", + "mime-type": "application/gzip", + "optional": true, + "type": "per-sample" + }, + "minimap2-index": { + "filepath": "BAMS/{{ alias }}.reads_aln_sort.bam.bai", + "title": "BAM index file (minimap2)", + "description": "Index file generated from mapping input reads to the reference.", + "mime-type": "application/octet-stream", + "optional": true, + "type": "per-sample" } } }