Merge branch 'CW-4613' into 'dev'

CW-4613 IGV config

Closes CW-4613

See merge request epi2melabs/workflows/wf-transcriptomes!181

.gitlab-ci.yml

@@ -53,7 +53,7 @@ docker-run:
                 "only_differential_expression", "differential_expression_gff3",
                 "ncbi_gzip", "ncbi_no_gene_id", "ensembl_with_versions",
                 "differential_expression_mouse", "no_ref_annotation",
-                "unstranded_annotation_error"
+                "unstranded_annotation_error", "igv", "igv_fai_gz"
             ]
     rules:
         # NOTE As we're overriding the rules block for the included docker-run
@@ -67,21 +67,21 @@ docker-run:
               NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \
                   --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
                   -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
-              NF_IGNORE_PROCESSES: preprocess_reads,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+              NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "isoforms_bam"
           variables:
             NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
             NF_WORKFLOW_OPTS: "--bam  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome-source reference-guided \
                   --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
                   -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
-            NF_IGNORE_PROCESSES: preprocess_reads,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+            NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "no_ref_annotation"
           variables:
               NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf  ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
               NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \
                   --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
                   -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
-              NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+              NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "differential_expression"
           variables:
               NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
@@ -91,7 +91,7 @@ docker-run:
                 --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
                 --direct_rna --minimap2_index_opts '-k 15'  --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
                 -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
-              NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+              NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "only_differential_expression"
           variables:
               NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/  &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
@@ -105,7 +105,7 @@ docker-run:
                 --sample_sheet test_data/sample_sheet.csv \
                 -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
               NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "differential_expression_gff3"
           variables:
@@ -120,7 +120,7 @@ docker-run:
                 --sample_sheet test_data/sample_sheet.csv \
                 -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config"
               NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "ncbi_gzip"
           variables:
@@ -135,7 +135,7 @@ docker-run:
                --sample_sheet test_data/sample_sheet.csv \
                -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config"
               NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome           
         - if: $MATRIX_NAME == "ncbi_no_gene_id"
           variables:
@@ -149,7 +149,7 @@ docker-run:
                 --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \
                 -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
               NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome      
         - if: $MATRIX_NAME == "ensembl_with_versions"
           variables:
@@ -163,7 +163,7 @@ docker-run:
                 --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \
                 -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
               NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
         - if: $MATRIX_NAME == "differential_expression_mouse"
           variables:
@@ -177,7 +177,7 @@ docker-run:
                 --transcriptome_assembly false --sample_sheet  ${CI_PROJECT_NAME}/data/differential_expression_mouse/sample_sheet.csv \
                 -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config"
               NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam    
         - if: $MATRIX_NAME == "unstranded_annotation_error"
           variables:
@@ -188,6 +188,37 @@ docker-run:
                 --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/unstranded_annotation.gtf \
                 --direct_rna --minimap2_index_opts '-k 15'  --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
                 -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
-              NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+              NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
               ASSERT_NEXTFLOW_FAILURE: "test_fail" # set to any non-zero length str to allow the nextflow CMD to fail
               ASSERT_NEXTFLOW_FAILURE_REXP: "In ref_annotation, transcript features must have a strand of either '+' or '-'"
+        - if: $MATRIX_NAME == "igv"
+          variables:
+              NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/  &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
+              NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
+                  --de_analysis \
+                  --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
+                --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \
+                --direct_rna --minimap2_index_opts '-k 15' \
+                --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \
+                --sample_sheet test_data/sample_sheet.csv \
+                --igv \
+                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
+              NF_IGNORE_PROCESSES: >
+                preprocess_reads,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
+                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
+        - if: $MATRIX_NAME == "igv_fai_gz"
+          variables:
+              NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf  ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C  ${CI_PROJECT_NAME}/data/ &&  wget -O  ${CI_PROJECT_NAME}/data/demo.nextflow.config  https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
+              NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
+                --fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
+                --transcriptome-source precomputed --de_analysis \
+                --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \
+                --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \
+                --direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \
+                --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \
+                --igv \
+                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
+              NF_IGNORE_PROCESSES: >
+                preprocess_reads,merge_transcriptomes,assemble_transcripts,
+                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome 
+        

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [unreleased]
+## Added
+- `--igv` parameter (default: false) for outputting IGV config allowing visualisation of read alignments in the EPI2ME App.
+- If required for IGV, reference indexes are output in to a `igv_reference` directory
+### Changed
+- BAMS are output in to a BAMS directory.
+
 ## [v1.3.0]
 ### Removed
 - Fusion detection subworkflow, as the functionality is not robust enough for general use at this time.

README.md

@@ -153,6 +153,7 @@ input_reads.fastq   ─── input_directory  ─── input_directory
 | Nextflow parameter name  | Type | Description | Help | Default |
 |--------------------------|------|-------------|------|---------|
 | out_dir | string | Directory for output of all user-facing files. |  | output |
+| igv | boolean | Visualize outputs in the EPI2ME IGV visualizer. | Enabling this option will visualize the output alignment files in the EPI2ME Desktop App IGV visualizer. | False |
 
 
 ### Sample Options
@@ -232,6 +233,11 @@ Output files may be aggregated including information for all samples or provided
 | Transcript counts filtered | de_analysis/filtered_transcript_counts_with_genes.tsv | Filtered transcript counts, used for differential transcript usage analysis. Includes a reference to the associated gene ID. | aggregated |
 | Transcript info table | {{ alias }}_transcripts_table.tsv | This file details each isoform that was reconstructed from the input reads. It contains a subset of columns from the .tmap output from [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) | per-sample |
 | Final non redundant transcriptome | de_analysis/final_non_redundant_transcriptome.fasta | Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. | aggregated |
+| Index of reference FASTA file | igv_reference/{{ ref_genome file }}.fai | Reference genome index of the FASTA file required for IGV config. | aggregated |
+| GZI index of the reference FASTA file | igv_reference/{{ ref_genome file }}.gzi | GZI Index of the reference FASTA file. | aggregated |
+| JSON configuration file for IGV browser | igv.json | JSON configuration file to be loaded in IGV for visualising alignments against the reference. | aggregated |
+| BAM file (minimap2) | BAMS/{{ alias }}.reads_aln_sorted.bam | BAM file generated from mapping input reads to the reference. | per-sample |
+| BAM index file (minimap2) | BAMS/{{ alias }}.reads_aln_sort.bam.bai | Index file generated from mapping input reads to the reference. | per-sample |
 
 
 
@@ -250,6 +256,8 @@ If the `transcriptome_source` parameter is "reference-guided" a transcriptome wi
 #### 3.1 Align reads with reference genome.
 The reference genome will be indexed and aligned using [Minimap2](https://github.com/lh3/minimap2). The output is sorted and converted to a BAM file using [Samtools](https://www.htslib.org/). Alignment stats are created from these using [Seqkit BAM](https://bioinf.shenwei.me/seqkit/usage/#bam).
 
+Additionally, the workflow will generate an IGV configuration file if `--igv` is selected. This file allows the user to view the aligned BAM in the EPI2ME Desktop Application in the Viewer tab.
+
 #### 3.2 Chunk BAM
 The aligned BAMs are split into chunks using the bundle_min_reads parameter (default: 50000).
 
@@ -316,6 +324,7 @@ The final component of this isoform analysis is a stage-wise statistical test us
 
 + If the workflow fails please run it with the demo data set to ensure the workflow itself is working. This will help us determine if the issue is related to the environment, input parameters or a bug.
 + See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/).
++ Renaming, moving or deleting the input BAM, reference genome or the output directory from the location provided at runtime will stop IGV in the EPI2ME Desktop app from loading.
 
 
 
@@ -323,6 +332,8 @@ The final component of this isoform analysis is a stage-wise statistical test us
 
 *Does the workflow support de novo assembly?* - Currently the workflow does not have a *de novo* mode.
 
+*Why is the IGV panel not showing?* - The workflow expects either an uncompressed or [`bgzip`](https://www.htslib.org/doc/bgzip.html)-compressed reference. If the user provides a reference compressed not with `bgzip`, the workflow will run to completion, but won't be able to generate the necessary indexes to visualize the outputs in IGV.
+
 If your question is not answered here, please report any issues or suggestions on the [github issues](https://github.com/epi2me-labs/wf-transcriptomes/issues) page or start a discussion on the [community](https://community.nanoporetech.com/).
 
 

docs/06_input_parameters.md

@@ -17,6 +17,7 @@
 | Nextflow parameter name  | Type | Description | Help | Default |
 |--------------------------|------|-------------|------|---------|
 | out_dir | string | Directory for output of all user-facing files. |  | output |
+| igv | boolean | Visualize outputs in the EPI2ME IGV visualizer. | Enabling this option will visualize the output alignment files in the EPI2ME Desktop App IGV visualizer. | False |
 
 
 ### Sample Options

docs/07_outputs.md

@@ -26,3 +26,8 @@ Output files may be aggregated including information for all samples or provided
 | Transcript counts filtered | de_analysis/filtered_transcript_counts_with_genes.tsv | Filtered transcript counts, used for differential transcript usage analysis. Includes a reference to the associated gene ID. | aggregated |
 | Transcript info table | {{ alias }}_transcripts_table.tsv | This file details each isoform that was reconstructed from the input reads. It contains a subset of columns from the .tmap output from [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) | per-sample |
 | Final non redundant transcriptome | de_analysis/final_non_redundant_transcriptome.fasta | Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. | aggregated |
+| Index of reference FASTA file | igv_reference/{{ ref_genome file }}.fai | Reference genome index of the FASTA file required for IGV config. | aggregated |
+| GZI index of the reference FASTA file | igv_reference/{{ ref_genome file }}.gzi | GZI Index of the reference FASTA file. | aggregated |
+| JSON configuration file for IGV browser | igv.json | JSON configuration file to be loaded in IGV for visualising alignments against the reference. | aggregated |
+| BAM file (minimap2) | BAMS/{{ alias }}.reads_aln_sorted.bam | BAM file generated from mapping input reads to the reference. | per-sample |
+| BAM index file (minimap2) | BAMS/{{ alias }}.reads_aln_sort.bam.bai | Index file generated from mapping input reads to the reference. | per-sample |

docs/08_pipeline_overview.md

@@ -10,6 +10,8 @@ If the `transcriptome_source` parameter is "reference-guided" a transcriptome wi
 #### 3.1 Align reads with reference genome.
 The reference genome will be indexed and aligned using [Minimap2](https://github.com/lh3/minimap2). The output is sorted and converted to a BAM file using [Samtools](https://www.htslib.org/). Alignment stats are created from these using [Seqkit BAM](https://bioinf.shenwei.me/seqkit/usage/#bam).
 
+Additionally, the workflow will generate an IGV configuration file if `--igv` is selected. This file allows the user to view the aligned BAM in the EPI2ME Desktop Application in the Viewer tab.
+
 #### 3.2 Chunk BAM
 The aligned BAMs are split into chunks using the bundle_min_reads parameter (default: 50000).
 

docs/09_troubleshooting.md

@@ -1,2 +1,3 @@
 + If the workflow fails please run it with the demo data set to ensure the workflow itself is working. This will help us determine if the issue is related to the environment, input parameters or a bug.
-+ See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/).
++ See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/).
++ Renaming, moving or deleting the input BAM, reference genome or the output directory from the location provided at runtime will stop IGV in the EPI2ME Desktop app from loading.

docs/10_FAQ.md

@@ -1,3 +1,5 @@
 *Does the workflow support de novo assembly?* - Currently the workflow does not have a *de novo* mode.
 
+*Why is the IGV panel not showing?* - The workflow expects either an uncompressed or [`bgzip`](https://www.htslib.org/doc/bgzip.html)-compressed reference. If the user provides a reference compressed not with `bgzip`, the workflow will run to completion, but won't be able to generate the necessary indexes to visualize the outputs in IGV.
+
 If your question is not answered here, please report any issues or suggestions on the [github issues](https://github.com/epi2me-labs/wf-transcriptomes/issues) page or start a discussion on the [community](https://community.nanoporetech.com/).

main.nf

@@ -10,6 +10,7 @@ import java.util.ArrayList;
 nextflow.enable.dsl = 2
 
 include { fastq_ingress; xam_ingress } from './lib/ingress'
+include { configure_igv } from './lib/common'
 include { reference_assembly } from './subworkflows/reference_assembly'
 include { differential_expression } from './subworkflows/differential_expression'
 
@@ -526,6 +527,50 @@ process check_annotation_strand {
 }
 
 
+// Process to create the faidx index
+process faidx {
+    // If the input file is gzipped, we need to emit the indexes for the input gzip file
+    // only. Therefore, this become redundant to be emitted as it won't be used by the
+    // IGV configuration, but only by internal processes together with the decompressed
+    // FASTA file. To avoid unnecessary emissions, we enable only if the input file is
+    // decompressed.
+    publishDir "${params.out_dir}/igv_reference", mode: 'copy', pattern: "*", enabled: !params.ref_genome.toLowerCase().endsWith("gz")
+    label "wf_common"
+    cpus 1
+    memory 4.GB
+    input:
+        path(ref)
+    output:
+        path("${ref}.fai")
+    script:
+    """
+    samtools faidx ${ref}
+    """
+}
+
+
+// Process to create the faidx indexes for a gzipped reference
+process gz_faidx {
+    publishDir "${params.out_dir}/igv_reference", mode: 'copy', pattern: "*"
+    label "wf_common"
+    cpus 1
+    memory 4.GB
+    // If a user provides a non-bgzipped file, the process won't
+    // generate the indexes. We should tolerate that, still avoid emitting
+    // the reference and simply have a broken IGV file.
+    // The gzi is not required to operate the workflow, so we actually tolerate any failure.
+    errorStrategy 'ignore'
+    input:
+        path(ref)
+    output:
+        tuple path("${ref}.fai"), path("${ref}.gzi")
+    script:
+    """
+    samtools faidx ${ref}
+    """
+}
+
+
 
 // workflow module
 workflow pipeline {
@@ -580,6 +625,8 @@ workflow pipeline {
 
      
         results = Channel.empty()
+        // Define BAM output Directory
+        String publish_bams = "BAMS"
         software_versions = getVersions()
         workflow_params = getParams()
         input_reads = reads.map{ meta, samples, stats -> [meta, samples]}
@@ -634,7 +681,9 @@ workflow pipeline {
 
 
             merge_gff = merge_gff_bundles.out.gff.map{ it -> it[1]}.collect()
-            results = results.concat(assembly.bam.map {sample_id, bam, bai -> [bam, bai]}.flatten())
+            // Output BAMS in a dedicated directory
+            bam_results = assembly.bam.map{ 
+                sample_id, bam, bai -> [bam, bai]}.flatten().map{ [it, publish_bams] }
         }
         else{
             gff_compare = OPTIONAL_FILE
@@ -723,6 +772,85 @@ workflow pipeline {
         }
 
         results.concat(workflow_params.map{ [it, null]})
+        // IGV config
+        if (params.transcriptome_source == "precomputed" && params.igv){
+            log.warn("IGV configuration does not work if transcriptome sources is set to `precomputed`.")
+        }
+        if (params.transcriptome_source != "precomputed" && params.igv){
+            is_compressed = file("${params.ref_genome}").extension == "gz"
+            String publish_ref = "igv_reference"
+            String current_dir = "${file('.').toUriString()}"
+            reference_genome = Channel.fromPath("${params.ref_genome}")
+            igv_ref = reference_genome | flatten | map { it -> "${it.toUriString()}" }
+            if (is_compressed){
+                // Define indexes names.
+                String input_fai_index = "${params.ref_genome}.fai"
+                String input_gzi_index = "${params.ref_genome}.gzi"
+                // Check whether the input gzref is indexed. If so, pass these as indexes.
+                // Otherwise, generate the gzip + fai indexes for the compressed reference.
+                if (file(input_fai_index).exists() && file(input_gzi_index).exists()){
+                    gzindexes = Channel.fromPath(input_fai_index)
+                    | mix(
+                        Channel.fromPath(input_gzi_index)
+                    )
+                    gz_igv = gzindexes | flatten | map { it -> "${it.toUriString()}" }
+                } else {
+                    gz_igv = gz_faidx(Channel.fromPath("${params.ref_genome}"))
+                    | flatten
+                    | map  { it -> "$current_dir/$params.out_dir/$publish_ref/${it.Name}" }
+                    gz_igv | ifEmpty{
+                        if (params.containsKey("igv") && params.igv){
+                            log.warn """\
+                                The input reference is compressed but not with bgzip, which is required to create an index.
+                                The workflow will proceed but it will not be possible to load the reference in the IGV Viewer.
+                                To use the IGV Viewer, provide an uncompressed, or bgzip compressed version of the input reference next time you run the workflow.
+                                """.stripIndent()
+                        }
+                    }
+                }
+            } else {
+                gzindexes = Channel.empty()
+                gz_igv = Channel.empty()
+            }
+
+            // Generate fai index if the file is either compressed, or if fai doesn't exists
+            if (!is_compressed && file("${params.ref_genome}.fai").exists()){
+                ref_idx = Channel.fromPath("${params.ref_genome}.fai")
+                igv_index = ref_idx | flatten | map { it -> "${it.toUriString()}" }
+            } else {
+                ref_idx = faidx(reference_genome)
+                igv_index = ref_idx | map { it -> "$current_dir/$params.out_dir/$publish_ref/${it.Name}" }
+            }
+
+            // get list of file names
+            // Absolute paths required for directories
+            igv_files = reads
+            | map { meta, sample, stats -> meta.alias }
+            | toSortedList
+            | map { list -> list.collect{
+                [
+                 "$current_dir/$params.out_dir/$publish_bams/${it}_reads_aln_sorted.bam",
+                 "$current_dir/$params.out_dir/$publish_bams/${it}_reads_aln_sorted.bam.bai"
+                ]
+            } }
+            | concat ( igv_index)
+            | flatten
+            | concat (igv_ref)
+            | concat (gz_igv)
+            | flatten
+            | collectFile(name: "file-names.txt", newLine: true, sort: false)
+
+            // configure IGV
+            igv_conf = configure_igv(
+                igv_files,
+                Channel.of(null), // igv locus
+                [displayMode: "SQUISHED", colorBy: "strand"], // bam extra opts
+                Channel.of(null), // vcf extra opts
+                )
+            
+            results = results.concat(igv_conf.map{ [it, null]})
+            results = results.concat(bam_results)
+      }
 
        
     emit:

nextflow.config

@@ -31,6 +31,7 @@ params {
     version = false
 
     monochrome_logs = false
+    igv = false
     validate_params = true
     show_hidden_params = false
     schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf'
@@ -95,7 +96,7 @@ params {
       ]
     agent = null
     container_sha = "shafb1e2372e1535f0b42891ed2c68ffdac2ca1d658"
-	common_sha = "shab540ba556d0d8c38bea8fec520f0bdedd9e59520"
+	common_sha = "shad399cf22079b5b153920ac39ee40095a677933f1"
     }
 }
 

nextflow_schema.json

@@ -110,6 +110,13 @@
                     "format": "directory-path",
                     "default": "output",
                     "description": "Directory for output of all user-facing files."
+                },
+                "igv": {
+                    "title": "Show in IGV",
+                    "type": "boolean",
+                    "default": false,
+                    "description": "Visualize outputs in the EPI2ME IGV visualizer.",
+                    "help_text": "Enabling this option will visualize the output alignment files in the EPI2ME Desktop App IGV visualizer."
                 }
             }
         },

output_definition.json

@@ -191,6 +191,46 @@
       "mime-type": "text/x-fasta",
       "optional": true,
       "type": "aggregated"
+    },
+    "reference-index": {
+      "filepath": "igv_reference/{{ ref_genome file }}.fai",
+      "title": "Index of reference FASTA file",
+      "description": "Reference genome index of the FASTA file required for IGV config.",
+      "mime-type": "text/tab-separated-values",
+      "optional": true,
+      "type": "aggregated"
+    },
+    "reference-gzi-index": {
+        "filepath": "igv_reference/{{ ref_genome file }}.gzi",
+        "title": "GZI index of the reference FASTA file",
+        "description": "GZI Index of the reference FASTA file.",
+        "mime-type": "application/octet-stream",
+        "optional": true,
+        "type": "aggregated"
+    },
+    "igv-config": {
+      "filepath": "igv.json",
+      "title": "JSON configuration file for IGV browser",
+      "description": "JSON configuration file to be loaded in IGV for visualising alignments against the reference.",
+      "mime-type": "text/json",
+      "optional": true,
+      "type": "aggregated"
+    },
+    "minimap2-bam": {
+      "filepath": "BAMS/{{ alias }}.reads_aln_sorted.bam",
+      "title": "BAM file (minimap2)",
+      "description": "BAM file generated from mapping input reads to the reference.",
+      "mime-type": "application/gzip",
+      "optional": true,
+      "type": "per-sample"
+    },
+    "minimap2-index": {
+      "filepath": "BAMS/{{ alias }}.reads_aln_sort.bam.bai",
+      "title": "BAM index file (minimap2)",
+      "description": "Index file generated from mapping input reads to the reference.",
+      "mime-type": "application/octet-stream",
+      "optional": true,
+      "type": "per-sample"
     }
   }
 }