From f9bd2ab574c5630a02bb297455707ce1b1c723e1 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Fri, 14 Mar 2025 19:06:47 +0000 Subject: [PATCH] Error message if ref_genome and ref_annotation mismatch CW-2965 --- .gitlab-ci.yml | 58 +++++++++++++++++++++++++++++++++++++++----------- CHANGELOG.md | 1 + main.nf | 53 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10c3c23..3832f5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,8 @@ docker-run: "only_differential_expression", "differential_expression_gff3", "ncbi_gzip", "ncbi_no_gene_id", "ensembl_with_versions", "differential_expression_mouse", "no_ref_annotation", - "unstranded_annotation_error", "igv", "igv_fai_gz" + "unstranded_annotation_error", "igv", "igv_fai_gz", + "mismatch_seqid_error","mismatch_seqid_warn" ] rules: # NOTE As we're overriding the rules block for the included docker-run @@ -69,21 +70,21 @@ docker-run: NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ -c ${CI_PROJECT_NAME}/data/mm2.config" - NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: preprocess_reads,validate_ref_annotation,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "isoforms_bam" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}" NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome_source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" - NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: preprocess_reads,validate_ref_annotation,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "no_ref_annotation" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" - NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome + NF_IGNORE_PROCESSES: run_gffcompare,validate_ref_annotation,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}" @@ -107,7 +108,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > - preprocess_reads,faidx,gz_faidx,merge_transcriptomes,merge_gff_bundles,assemble_transcripts,decompress_annotation,decompress_ref, + preprocess_reads,faidx,validate_ref_annotation,gz_faidx,merge_transcriptomes,merge_gff_bundles,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_gff3" variables: @@ -122,7 +123,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > - preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, + preprocess_reads,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ncbi_gzip" variables: @@ -150,7 +151,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > - preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, + preprocess_reads,faidx,validate_ref_annotation,gz_faidx,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ensembl_with_versions" variables: @@ -164,7 +165,7 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > - preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, + preprocess_reads,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_mouse" variables: @@ -178,7 +179,7 @@ docker-run: --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression_mouse/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > - preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation, + preprocess_reads,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam - if: $MATRIX_NAME == "unstranded_annotation_error" variables: @@ -206,7 +207,7 @@ docker-run: -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > preprocess_reads,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, - build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "igv_fai_gz" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}" @@ -220,6 +221,39 @@ docker-run: --igv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" NF_IGNORE_PROCESSES: > - preprocess_reads,merge_transcriptomes,assemble_transcripts, + preprocess_reads,validate_ref_annotation,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome,faidx,gz_faidx - + - if: $MATRIX_NAME == "mismatch_seqid_error" + variables: + NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}" + NF_WORKFLOW_OPTS: + "--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \ + --de_analysis \ + --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/hg38_chr20.fa \ + --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \ + --direct_rna \ + --sample_sheet test_data/sample_sheet.csv \ + -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" + NF_IGNORE_PROCESSES: > + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, + build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + ASSERT_NEXTFLOW_FAILURE: 1 + ASSERT_NEXTFLOW_FAILURE_REXP: Seqid mismatch found between the provided ref_annotation + - if: $MATRIX_NAME == "mismatch_seqid_warn" + variables: + NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}" + NF_WORKFLOW_OPTS: + "--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \ + --de_analysis \ + --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \ + --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \ + --direct_rna \ + --sample_sheet test_data/sample_sheet.csv --minimap2_index_opts '-w 25' \ + -c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config" + NF_IGNORE_PROCESSES: > + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts, + build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + AFTER_NEXTFLOW_CMD: "grep 'are only present in the reference genome' '.nextflow.log'" + + + diff --git a/CHANGELOG.md b/CHANGELOG.md index ae1e4ba..06d705e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased. - Updated recommended memory requirement. - Updated project description. +- A common user issue is providing a ref_annotation and ref_genome parameter that have mismatched reference IDs, which causes the DE_analysis to fail. The workflow will now do an upfront check and give an error message if no overlap is found or a warning if some IDs are present in one file but not in the other. - Reconciled workflow with wf-template v5.5.0. ### Fixed - `all_gene_counts.tsv` contained the DE counts results. diff --git a/main.nf b/main.nf index 0f4eb97..c3ff2e8 100644 --- a/main.nf +++ b/main.nf @@ -69,6 +69,54 @@ process decompress_ref { """ } +process validate_ref_annotation { + label "isoforms" + cpus 1 + memory "2 GB" + input: + path "annotation.gtf" + path "reference.fasta" + output: + stdout + // Checks for overlap between seq_id column in annotation gtf and fasta reference ID's + // If no overlap is found exit + // Partial overlap (eg. user supplies genes/contigs of interest in annotation but the genome sequence) - warning + script: + """ + grep -v '^#' annotation.gtf | cut -f1 | sort -u > seq_ids.txt + awk '/^>/ {print substr(\$1,2)}' reference.fasta | sort -u > ref_ids.txt + matches=\$(comm -12 seq_ids.txt ref_ids.txt) + only_in_annotation=\$(comm -23 seq_ids.txt ref_ids.txt) + only_in_reference=\$(comm -13 seq_ids.txt ref_ids.txt) + if [[ -z "\$matches" ]]; then + echo " + ERROR: Seqid mismatch found between the provided ref_annotation (GTF/GFF) + file and ref_genome (FASTA). + For the reference guided differential expression subworkflow they must overlap. + " >&2 + echo "Annotation ID examples:" + head -n 5 seq_ids.txt + echo "Reference ID examples:" + head -n 5 ref_ids.txt + echo "We recommend getting both files from the same source. + eg. both from Ensembl or both from NCBI. + Alternatively provide a pre-computed transcriptome using the ref_transcriptome parameter + See the README for more details on which inputs are supported." + exit 78 + fi + if [[ -n "\$only_in_annotation" ]]; then + echo "Warning: Some sequence IDs are only present in the reference annotation and not the + reference genome so will not be used in downstream analysis eg." + echo "\$only_in_annotation" | head -n 5 + fi + if [[ -n "\$only_in_reference" ]]; then + echo "Warning: Some FASTA reference IDs are only present in the reference genome + and not the reference annotation so will not be used in downstream analysis eg." + echo "\$only_in_reference" | head -n 5 + fi + """ +} + process decompress_annotation { label "isoforms" @@ -695,6 +743,11 @@ workflow pipeline { stdoutput } if (!params.ref_transcriptome){ + validate_ref_annotation(ref_annotation, ref_genome).map { stdoutput -> + if (stdoutput) { + log.warn(stdoutput) + } + } merge_transcriptomes(run_gffcompare.output.gtf.collect(), ref_annotation, ref_genome) transcriptome = merge_transcriptomes.out.fasta gtf = merge_transcriptomes.out.gtf