From c8c59229652cdc1ca27539ef13965488dee72cdd Mon Sep 17 00:00:00 2001 From: Neil Horner Date: Fri, 13 Dec 2024 11:51:04 +0000 Subject: [PATCH] Access to undefined channel output bug CW-5398 --- .gitlab-ci.yml | 29 ++++++++++++++--------------- CHANGELOG.md | 3 +-- bin/workflow_glue/report.py | 2 +- main.nf | 11 ++++++++--- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49a524c..253f71c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ - --transcriptome-source ${CI_PROJECT_NAME}/data/reference-guided \ + --transcriptome_source reference-guided \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" @@ -66,21 +66,21 @@ docker-run: - if: $MATRIX_NAME == "isoforms" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config - NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \ + NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "isoforms_bam" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config - NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome-source reference-guided \ + NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome_source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "no_ref_annotation" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config - NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \ + NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome @@ -89,7 +89,7 @@ docker-run: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --de_analysis \ - --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \ + --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome_source reference-guided \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" @@ -98,7 +98,7 @@ docker-run: variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ - --transcriptome-source precomputed \ + --transcriptome_source precomputed \ --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \ @@ -107,13 +107,13 @@ docker-run: --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > - preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, - build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + preprocess_reads,faidx,gz_faidx,merge_transcriptomes,merge_gff_bundles,assemble_transcripts,decompress_annotation,decompress_ref, + build_minimap_index,get_transcriptome,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_gff3" variables: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ - --transcriptome-source precomputed \ + --transcriptome_source precomputed \ --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff3 \ @@ -129,7 +129,6 @@ docker-run: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \ - --transcriptome-source precomputed \ --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \ @@ -144,7 +143,7 @@ docker-run: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \ - --transcriptome-source precomputed --de_analysis \ + --transcriptome_source precomputed --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \ --direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \ @@ -172,7 +171,7 @@ docker-run: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz -C ${CI_PROJECT_NAME}/data/&& wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq ${CI_PROJECT_NAME}/data/differential_expression_mouse/differential_expression_fastq \ - --transcriptome-source precomputed --de_analysis \ + --transcriptome_source precomputed --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_mouse/GRCm39.genome.fa.gz \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_mouse/gencode.vM33.annotation.gtf \ --direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_mouse/gencode.vM33.transcripts.fa.gz \ @@ -186,7 +185,7 @@ docker-run: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --de_analysis \ - --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \ + --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome_source reference-guided \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/unstranded_annotation.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" @@ -213,7 +212,7 @@ docker-run: NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \ - --transcriptome-source precomputed --de_analysis \ + --transcriptome_source precomputed --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \ --direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \ @@ -222,5 +221,5 @@ docker-run: -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts, - build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome + build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome,faidx,gz_faidx diff --git a/CHANGELOG.md b/CHANGELOG.md index 03faa5c..7b909af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -### Fixed -- Bug that led to incorrect gene_id being assigned in the DE plots. +- Fix v1.4.0 bug; access to undefined channel output bug when using precomputed transcriptome. ## [v1.5.0] ### Updated diff --git a/bin/workflow_glue/report.py b/bin/workflow_glue/report.py index 6c910f0..8ad510e 100644 --- a/bin/workflow_glue/report.py +++ b/bin/workflow_glue/report.py @@ -250,7 +250,7 @@ def transcriptome_summary(report, summaries_dir): tabs = Tabs() data = {} # Load all the dataframes upfront to get sample_id for sorting. - for summ_file in summaries_dir.glob('summary_*.tsv'): + for summ_file in summaries_dir.glob('summary_*.pkl'): with open(summ_file, 'rb') as fh: summ = pickle.load(fh) sample_id = summ['sample_id'] diff --git a/main.nf b/main.nf index 228903e..4ef6872 100644 --- a/main.nf +++ b/main.nf @@ -404,7 +404,7 @@ process makeReport { path gff_annotation, stageAs: "gff_annotation/*" path de_report, stageAs: "de_report/*" path isoforms_table, stageAs: "isoforms_table/*" - path "transcriptome_summary/summary_*.tsv" + path transcriptome_summary, stageAs: "transcriptome_summary/summary_*.pkl" output: path ("wf-transcriptomes-*.html"), emit: report @@ -422,6 +422,7 @@ process makeReport { String aln_stats_opts = aln_stats.fileName.name == OPTIONAL_FILE.name ? "" : "--alignment_stats aln_stats/" String pychop_opts = pychopper.fileName.name == OPTIONAL_FILE.name ? "" : "--pychop_report pychopper_report/" String iso_table_opts = isoforms_table.fileName.name == OPTIONAL_FILE.name ? "" : "--isoform_table isoforms_table/" + String tr_summary_opts = transcriptome_summary.fileName.name == OPTIONAL_FILE.name ? "" : "--transcriptome_summary transcriptome_summary/" """ echo '${metadata}' > metadata.json workflow-glue report \ @@ -438,7 +439,7 @@ process makeReport { $gffcmp_opts \ --isoform_table_nrows ${params.isoform_table_nrows} \ $de_report_opts \ - --transcriptome_summary transcriptome_summary/ + $tr_summary_opts """ } @@ -633,6 +634,7 @@ workflow pipeline { full_len_reads = input_reads.map{ meta, reads -> [meta.alias, reads]} pychopper_report = OPTIONAL_FILE } + if (params.transcriptome_source != "precomputed"){ build_minimap_index(ref_genome) log.info("Doing reference based transcript analysis") @@ -645,6 +647,8 @@ workflow pipeline { assemble_transcripts(split_bam.out.bundles.flatMap(map_sample_ids_cls).combine(ref_annotation),use_ref_ann) merge_gff_bundles(assemble_transcripts.out.gff_bundles.groupTuple()) + transcriptome_summary = merge_gff_bundles.out.summary.map {it[1]}.collect() + // only run gffcompare if ref annotation provided. Otherwise create optional files and channels if (params.ref_annotation){ run_gffcompare(merge_gff_bundles.out.gff, ref_annotation) @@ -679,6 +683,7 @@ workflow pipeline { isoforms_table = OPTIONAL_FILE merge_gff = OPTIONAL_FILE assembly_stats = OPTIONAL_FILE + transcriptome_summary = OPTIONAL_FILE use_ref_ann = false } if (params.de_analysis){ @@ -732,7 +737,7 @@ workflow pipeline { merge_gff, de_report, isoforms_table, - merge_gff_bundles.out.summary.map {it[1]}.collect()) + transcriptome_summary) report = makeReport.out.report