Merge branch 'CW-2965' into 'dev'

Error message if ref_genome and ref_annotation mismatch CW-2965 Closes CW-2965 See merge request epi2melabs/workflows/wf-transcriptomes!202
2025-03-14 19:06:47 +00:00 · 2025-03-14 19:06:47 +00:00 · 9900143122
commit 9900143122
parent 9ecd1345e9 f9bd2ab574
3 changed files with 100 additions and 12 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -55,7 +55,8 @@ docker-run:
                "only_differential_expression", "differential_expression_gff3",
                "ncbi_gzip", "ncbi_no_gene_id", "ensembl_with_versions",
                "differential_expression_mouse", "no_ref_annotation",
-                "unstranded_annotation_error", "igv", "igv_fai_gz"
+                "unstranded_annotation_error", "igv", "igv_fai_gz",
+                "mismatch_seqid_error","mismatch_seqid_warn"
            ]
    rules:
        # NOTE As we're overriding the rules block for the included docker-run
@ -69,21 +70,21 @@ docker-run:
              NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \
                  --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
                  -c ${CI_PROJECT_NAME}/data/mm2.config"
-              NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+              NF_IGNORE_PROCESSES: preprocess_reads,validate_ref_annotation,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "isoforms_bam"
          variables:
            NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
            NF_WORKFLOW_OPTS: "--bam  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome_source reference-guided \
                  --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
                  -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
-            NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+            NF_IGNORE_PROCESSES: preprocess_reads,validate_ref_annotation,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "no_ref_annotation"
          variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf  ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
              NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \
                  --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
                  -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
-              NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
+              NF_IGNORE_PROCESSES: run_gffcompare,validate_ref_annotation,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "differential_expression"
          variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
@ -107,7 +108,7 @@ docker-run:
                --sample_sheet test_data/sample_sheet.csv \
                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
-                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,merge_gff_bundles,assemble_transcripts,decompress_annotation,decompress_ref,
+                preprocess_reads,faidx,validate_ref_annotation,gz_faidx,merge_transcriptomes,merge_gff_bundles,assemble_transcripts,decompress_annotation,decompress_ref,
                build_minimap_index,get_transcriptome,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "differential_expression_gff3"
          variables:
@ -122,7 +123,7 @@ docker-run:
                --sample_sheet test_data/sample_sheet.csv \
                -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
-                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
+                preprocess_reads,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "ncbi_gzip"
          variables:
@ -150,7 +151,7 @@ docker-run:
                --sample_sheet test_data/sample_sheet.csv \
                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
-                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
+                preprocess_reads,faidx,validate_ref_annotation,gz_faidx,merge_transcriptomes,assemble_transcripts,
                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome      
        - if: $MATRIX_NAME == "ensembl_with_versions"
          variables:
@ -164,7 +165,7 @@ docker-run:
                --sample_sheet test_data/sample_sheet.csv \
                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
-                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
+                preprocess_reads,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "differential_expression_mouse"
          variables:
@ -178,7 +179,7 @@ docker-run:
                --sample_sheet  ${CI_PROJECT_NAME}/data/differential_expression_mouse/sample_sheet.csv \
                -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
-                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,
+                preprocess_reads,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,
                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam    
        - if: $MATRIX_NAME == "unstranded_annotation_error"
          variables:
@ -206,7 +207,7 @@ docker-run:
                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
                preprocess_reads,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
-                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
+                build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
        - if: $MATRIX_NAME == "igv_fai_gz"
          variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf  ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C  ${CI_PROJECT_NAME}/data/ &&  wget -O  ${CI_PROJECT_NAME}/data/demo.nextflow.config  https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
@ -220,6 +221,39 @@ docker-run:
                --igv \
                -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
              NF_IGNORE_PROCESSES: >
-                preprocess_reads,merge_transcriptomes,assemble_transcripts,
+                preprocess_reads,validate_ref_annotation,merge_transcriptomes,assemble_transcripts,
                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome,faidx,gz_faidx 
-        
+        - if: $MATRIX_NAME == "mismatch_seqid_error"
+          variables:
+              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O  ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf  ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C  ${CI_PROJECT_NAME}/data/ &&  wget -O  ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
+              NF_WORKFLOW_OPTS:
+                  "--fastq   ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
+                  --de_analysis \
+                  --ref_genome  ${CI_PROJECT_NAME}/data/differential_expression_ncbi/hg38_chr20.fa \
+                  --ref_annotation  ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \
+                  --direct_rna \
+                  --sample_sheet test_data/sample_sheet.csv \
+                  -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
+              NF_IGNORE_PROCESSES: >
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
+                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome           
+              ASSERT_NEXTFLOW_FAILURE: 1
+              ASSERT_NEXTFLOW_FAILURE_REXP: Seqid mismatch found between the provided ref_annotation
+        - if: $MATRIX_NAME == "mismatch_seqid_warn"
+          variables:
+              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O  ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf  ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C  ${CI_PROJECT_NAME}/data/ &&  wget -O  ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
+              NF_WORKFLOW_OPTS:
+                  "--fastq   ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
+                  --de_analysis \
+                  --ref_genome  ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \
+                  --ref_annotation  ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \
+                  --direct_rna \
+                  --sample_sheet test_data/sample_sheet.csv --minimap2_index_opts '-w 25' \
+                  -c  ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
+              NF_IGNORE_PROCESSES: >
+                preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
+                build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome           
+              AFTER_NEXTFLOW_CMD: "grep 'are only present in the reference genome' '.nextflow.log'"
+
+
+
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased.
 - Updated recommended memory requirement.
 - Updated project description.
+- A common user issue is providing a ref_annotation and ref_genome parameter that have mismatched reference IDs, which causes the DE_analysis to fail. The workflow will now do an upfront check and give an error message if no overlap is found or a warning if some IDs are present in one file but not in the other.
 - Reconciled workflow with wf-template v5.5.0.
 ### Fixed
 - `all_gene_counts.tsv` contained the DE counts results.
--- a/main.nf
+++ b/main.nf
@ -69,6 +69,54 @@ process decompress_ref {
    """
 }

+process validate_ref_annotation {
+    label "isoforms"
+    cpus 1
+    memory "2 GB"
+    input:
+        path "annotation.gtf"
+        path "reference.fasta"
+    output: 
+        stdout
+    // Checks for overlap between seq_id column in annotation gtf and fasta reference ID's
+    // If no overlap is found exit
+    // Partial overlap (eg. user supplies genes/contigs of interest in annotation but the genome sequence) - warning
+    script:
+    """
+    grep -v '^#' annotation.gtf | cut -f1  | sort -u > seq_ids.txt
+    awk '/^>/ {print substr(\$1,2)}' reference.fasta | sort -u > ref_ids.txt
+    matches=\$(comm -12 seq_ids.txt ref_ids.txt)
+    only_in_annotation=\$(comm -23 seq_ids.txt ref_ids.txt)
+    only_in_reference=\$(comm -13 seq_ids.txt ref_ids.txt)
+    if [[ -z "\$matches" ]]; then
+        echo "
+        ERROR: Seqid mismatch found between the provided ref_annotation (GTF/GFF)
+        file and ref_genome (FASTA).
+        For the reference guided differential expression subworkflow they must overlap.
+        " >&2
+        echo "Annotation ID examples:"
+        head -n 5 seq_ids.txt
+        echo "Reference ID examples:"
+        head -n 5 ref_ids.txt
+        echo "We recommend getting both files from the same source.
+        eg. both from Ensembl or both from NCBI.
+        Alternatively provide a pre-computed transcriptome using the ref_transcriptome parameter
+        See the README for more details on which inputs are supported."
+        exit 78 
+    fi
+    if [[ -n "\$only_in_annotation" ]]; then
+        echo "Warning: Some sequence IDs are only present in the reference annotation and not the
+             reference genome so will not be used in downstream analysis eg."
+        echo "\$only_in_annotation" | head -n 5
+    fi
+    if [[ -n "\$only_in_reference" ]]; then
+        echo "Warning: Some FASTA reference IDs are only present in the reference genome
+            and not the reference annotation so will not be used in downstream analysis eg."
+        echo "\$only_in_reference" | head -n 5
+    fi
+    """
+}
+

 process decompress_annotation {
    label "isoforms"
@ -695,6 +743,11 @@ workflow pipeline {
                    stdoutput
                }
            if (!params.ref_transcriptome){
+                validate_ref_annotation(ref_annotation, ref_genome).map { stdoutput ->
+                    if (stdoutput) {
+                        log.warn(stdoutput)
+                    }
+                }
                merge_transcriptomes(run_gffcompare.output.gtf.collect(), ref_annotation, ref_genome)
                transcriptome = merge_transcriptomes.out.fasta
                gtf = merge_transcriptomes.out.gtf