Merge branch 'template-update' into 'dev'
Template update See merge request epi2melabs/workflows/wf-transcriptomes!166
This commit is contained in:
Sarah Griffiths
commit
f85e039f98
14
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
14
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
@ -45,13 +45,19 @@ body:
|
||||
label: Workflow Execution
|
||||
description: Where are you running the workflow?
|
||||
options:
|
||||
- EPI2ME Desktop application
|
||||
- Command line
|
||||
- EPI2ME cloud agent
|
||||
- EPI2ME Desktop (Local)
|
||||
- EPI2ME Desktop (Cloud)
|
||||
- Command line (Local)
|
||||
- Command line (Cluster)
|
||||
- Other (please describe)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: other-workflow-execution
|
||||
attributes:
|
||||
label: Other workflow execution
|
||||
description: If "Other", please describe
|
||||
placeholder: Tell us where / how you are running the workflow.
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
|
||||
128
.gitlab-ci.yml
128
.gitlab-ci.yml
@ -4,14 +4,14 @@ include:
|
||||
file: "wf-containers.yaml"
|
||||
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \
|
||||
--de_analysis --ref_genome differential_expression/hg38_chr20.fa \
|
||||
--transcriptome-source reference-guided \
|
||||
--ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \
|
||||
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \
|
||||
--jaffal_refBase differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \
|
||||
-c demo.nextflow.config"
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
||||
--transcriptome-source ${CI_PROJECT_NAME}/data/reference-guided \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
|
||||
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
|
||||
--jaffal_refBase ${CI_PROJECT_NAME}/data/differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
CI_FLAVOUR: "new"
|
||||
|
||||
macos-run:
|
||||
@ -63,120 +63,120 @@ docker-run:
|
||||
when: never
|
||||
- if: $MATRIX_NAME == "isoforms"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ERR6053095_chr20.fastq --transcriptome-source reference-guided \
|
||||
--ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
|
||||
-c demo.nextflow.config"
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "no_ref_annotation"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ERR6053095_chr20.fastq --transcriptome-source reference-guided \
|
||||
--ref_genome chr20/hg38_chr20.fa \
|
||||
-c demo.nextflow.config"
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: run_gffcompare,preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "fusions"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB --fastq ERR6053095_chr20.fastq --transcriptome-source reference-guided \
|
||||
--ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \
|
||||
--jaffal_refBase chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \
|
||||
-c demo.nextflow.config"
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB --fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome-source reference-guided \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf \
|
||||
--jaffal_refBase ${CI_PROJECT_NAME}/data/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "differential_expression"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \
|
||||
--ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \
|
||||
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
|
||||
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "only_differential_expression"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--transcriptome-source precomputed \
|
||||
--de_analysis \
|
||||
--ref_genome differential_expression/hg38_chr20.fa \
|
||||
--ref_annotation differential_expression/gencode.v22.annotation.chr20.gff \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \
|
||||
--direct_rna --minimap2_index_opts '-k 15' \
|
||||
--ref_transcriptome differential_expression/ref_transcriptome.fasta \
|
||||
--ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "differential_expression_gff3"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--transcriptome-source precomputed \
|
||||
--de_analysis \
|
||||
--ref_genome differential_expression/hg38_chr20.fa \
|
||||
--ref_annotation differential_expression/gencode.v22.annotation.chr20.gff3 \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff3 \
|
||||
--direct_rna --minimap2_index_opts '-k 15' \
|
||||
--ref_transcriptome differential_expression/ref_transcriptome.fasta \
|
||||
--ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "ncbi_gzip"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf differential_expression_ncbi.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq differential_expression_ncbi/differential_expression_fastq \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
--transcriptome-source precomputed \
|
||||
--de_analysis \
|
||||
--ref_genome differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \
|
||||
--ref_annotation differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \
|
||||
--direct_rna --minimap2_index_opts '-w 25' \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "ncbi_no_gene_id"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf differential_expression_ncbi.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq differential_expression_ncbi/differential_expression_fastq \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
--transcriptome-source precomputed --de_analysis \
|
||||
--ref_genome differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \
|
||||
--ref_annotation differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \
|
||||
--direct_rna --ref_transcriptome differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \
|
||||
--transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "ensembl_with_versions"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq differential_expression/differential_expression_fastq \
|
||||
--transcriptome-source precomputed --de_analysis \
|
||||
--ref_genome differential_expression/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz \
|
||||
--ref_annotation differential_expression/Homo_sapiens.GRCh38.109.gtf.gz \
|
||||
--direct_rna --ref_transcriptome differential_expression/Homo_sapiens.GRCh38.cdna.all.fa.gz \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/Homo_sapiens.GRCh38.109.gtf.gz \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/Homo_sapiens.GRCh38.cdna.all.fa.gz \
|
||||
--transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "differential_expression_mouse"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: wget -O differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf differential_expression_mouse.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz -C ${CI_PROJECT_NAME}/data/&& wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq differential_expression_mouse/differential_expression_fastq \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_mouse/differential_expression_fastq \
|
||||
--transcriptome-source precomputed --de_analysis \
|
||||
--ref_genome differential_expression_mouse/GRCm39.genome.fa.gz \
|
||||
--ref_annotation differential_expression_mouse/gencode.vM33.annotation.gtf \
|
||||
--direct_rna --ref_transcriptome differential_expression_mouse/gencode.vM33.transcripts.fa.gz \
|
||||
--transcriptome_assembly false --sample_sheet differential_expression_mouse/sample_sheet.csv \
|
||||
-c demo.nextflow.config"
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_mouse/GRCm39.genome.fa.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_mouse/gencode.vM33.annotation.gtf \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_mouse/gencode.vM33.transcripts.fa.gz \
|
||||
--transcriptome_assembly false --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression_mouse/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam
|
||||
|
||||
@ -8,7 +8,7 @@ repos:
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
additional_dependencies:
|
||||
- epi2melabs>=0.0.51
|
||||
- epi2melabs>=0.0.53
|
||||
- id: build_models
|
||||
name: build_models
|
||||
entry: datamodel-codegen --strict-nullable --base-class workflow_glue.results_schema_helpers.BaseModel --use-schema-description --disable-timestamp --input results_schema.yml --input-file-type openapi --output bin/workflow_glue/results_schema.py
|
||||
|
||||
14
README.md
14
README.md
@ -122,6 +122,13 @@ input_reads.fastq ─── input_directory ─── input_directory
|
||||
| analyse_unclassified | boolean | Analyse unclassified reads from input directory. By default the workflow will not process reads in the unclassified directory. | If selected and if the input is a multiplex directory the workflow will also process the unclassified directory. | False |
|
||||
|
||||
|
||||
### Output Options
|
||||
|
||||
| Nextflow parameter name | Type | Description | Help | Default |
|
||||
|--------------------------|------|-------------|------|---------|
|
||||
| out_dir | string | Directory for output of all user-facing files. | | output |
|
||||
|
||||
|
||||
### Sample Options
|
||||
|
||||
| Nextflow parameter name | Type | Description | Help | Default |
|
||||
@ -174,13 +181,6 @@ input_reads.fastq ─── input_directory ─── input_directory
|
||||
| isoform_table_nrows | integer | Maximum rows to dispay in the isoform report table | | 5000 |
|
||||
|
||||
|
||||
### Miscellaneous Options
|
||||
|
||||
| Nextflow parameter name | Type | Description | Help | Default |
|
||||
|--------------------------|------|-------------|------|---------|
|
||||
| disable_ping | boolean | Enable to prevent sending a workflow ping. | | False |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -8,9 +8,10 @@ import pysam
|
||||
from .util import get_named_logger, wf_parser # noqa: ABS101
|
||||
|
||||
|
||||
def get_sq_lines(xam_file):
|
||||
"""Extract the `@SQ` lines from the header of a XAM file."""
|
||||
return pysam.AlignmentFile(xam_file, check_sq=False).header["SQ"]
|
||||
def get_sq_hd_lines(xam_file):
|
||||
"""Extract the `@SQ` and `@HD` lines from the header of a XAM file."""
|
||||
alignments = pysam.AlignmentFile(xam_file, check_sq=False)
|
||||
return alignments.header["SQ"], alignments.header["HD"]
|
||||
|
||||
|
||||
def main(args):
|
||||
@ -27,10 +28,17 @@ def main(args):
|
||||
# Set `is_unaligned` accordingly. If there are mixed headers (either with some files
|
||||
# containing `@SQ` lines and some not or with different files containing different
|
||||
# `@SQ` lines), set `mixed_headers` to `True`.
|
||||
# Also check if there is the SO line, to validate whether the file is (un)sorted.
|
||||
first_sq_lines = None
|
||||
mixed_headers = False
|
||||
sorted_xam = False
|
||||
for xam_file in target_files:
|
||||
sq_lines = get_sq_lines(xam_file)
|
||||
sq_lines, hd_lines = get_sq_hd_lines(xam_file)
|
||||
# Check if it is sorted.
|
||||
# When there is more than one BAM, merging/sorting
|
||||
# will happen regardless of this flag.
|
||||
if hd_lines.get('SO') == 'coordinate':
|
||||
sorted_xam = True
|
||||
if first_sq_lines is None:
|
||||
# this is the first file
|
||||
first_sq_lines = sq_lines
|
||||
@ -46,7 +54,9 @@ def main(args):
|
||||
# write `is_unaligned` and `mixed_headers` out so that they can be set as env.
|
||||
# variables
|
||||
sys.stdout.write(
|
||||
f"IS_UNALIGNED={int(is_unaligned)};MIXED_HEADERS={int(mixed_headers)}"
|
||||
f"IS_UNALIGNED={int(is_unaligned)};" +
|
||||
f"MIXED_HEADERS={int(mixed_headers)};" +
|
||||
f"IS_SORTED={int(sorted_xam)}"
|
||||
)
|
||||
logger.info(f"Checked (u)BAM headers in '{args.input_path}'.")
|
||||
|
||||
|
||||
@ -49,6 +49,21 @@ def main(args):
|
||||
try:
|
||||
encoding = determine_codec(args.sample_sheet)
|
||||
with open(args.sample_sheet, "r", encoding=encoding) as f:
|
||||
try:
|
||||
# Excel files don't throw any error until here
|
||||
csv.Sniffer().sniff(f.readline())
|
||||
f.seek(0) # return to initial position again
|
||||
except Exception as e:
|
||||
# Excel fails with UniCode error
|
||||
sys.stdout.write(
|
||||
"The sample sheet doesn't seem to be a CSV file.\n"
|
||||
"The sample sheet has to be a CSV file.\n"
|
||||
"Please verify that the sample sheet is a CSV file.\n"
|
||||
f"Parsing error: {e}"
|
||||
)
|
||||
|
||||
sys.exit()
|
||||
|
||||
csv_reader = csv.DictReader(f)
|
||||
n_row = 0
|
||||
for row in csv_reader:
|
||||
|
||||
43
bin/workflow_glue/check_xam_index.py
Executable file
43
bin/workflow_glue/check_xam_index.py
Executable file
@ -0,0 +1,43 @@
|
||||
"""Validate a single (u)BAM file index."""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pysam
|
||||
|
||||
from .util import get_named_logger, wf_parser # noqa: ABS101
|
||||
|
||||
|
||||
def validate_xam_index(xam_file):
|
||||
"""Use fetch to validate the index.
|
||||
|
||||
Invalid indexes will fail the call with a ValueError:
|
||||
ValueError: fetch called on bamfile without index
|
||||
"""
|
||||
alignments = pysam.AlignmentFile(xam_file, check_sq=False)
|
||||
try:
|
||||
alignments.fetch()
|
||||
has_valid_index = True
|
||||
except ValueError:
|
||||
has_valid_index = False
|
||||
return has_valid_index
|
||||
|
||||
|
||||
def main(args):
|
||||
"""Run the entry point."""
|
||||
logger = get_named_logger("checkBamIdx")
|
||||
|
||||
# Check if a XAM has a valid index
|
||||
has_valid_index = validate_xam_index(args.input_xam)
|
||||
# write `has_valid_index` out so that they can be set as env.
|
||||
sys.stdout.write(
|
||||
f"HAS_VALID_INDEX={int(has_valid_index)}"
|
||||
)
|
||||
logger.info(f"Checked (u)BAM index for: '{args.input_xam}'.")
|
||||
|
||||
|
||||
def argparser():
|
||||
"""Argument parser for entrypoint."""
|
||||
parser = wf_parser("check_bam_headers")
|
||||
parser.add_argument("input_xam", type=Path, help="Path to target XAM")
|
||||
return parser
|
||||
@ -11,6 +11,13 @@
|
||||
| analyse_unclassified | boolean | Analyse unclassified reads from input directory. By default the workflow will not process reads in the unclassified directory. | If selected and if the input is a multiplex directory the workflow will also process the unclassified directory. | False |
|
||||
|
||||
|
||||
### Output Options
|
||||
|
||||
| Nextflow parameter name | Type | Description | Help | Default |
|
||||
|--------------------------|------|-------------|------|---------|
|
||||
| out_dir | string | Directory for output of all user-facing files. | | output |
|
||||
|
||||
|
||||
### Sample Options
|
||||
|
||||
| Nextflow parameter name | Type | Description | Help | Default |
|
||||
@ -63,10 +70,3 @@
|
||||
| isoform_table_nrows | integer | Maximum rows to dispay in the isoform report table | | 5000 |
|
||||
|
||||
|
||||
### Miscellaneous Options
|
||||
|
||||
| Nextflow parameter name | Type | Description | Help | Default |
|
||||
|--------------------------|------|-------------|------|---------|
|
||||
| disable_ping | boolean | Enable to prevent sending a workflow ping. | | False |
|
||||
|
||||
|
||||
|
||||
355
lib/ingress.nf
355
lib/ingress.nf
@ -24,7 +24,8 @@ def is_target_file(Path file, List extensions) {
|
||||
|
||||
|
||||
/**
|
||||
* Take a channel of the shape `[meta, reads, path-to-stats-dir | null]` and extract the
|
||||
* Take a channel of the shape `[meta, reads, path-to-stats-dir | null]` (or
|
||||
* `[meta, [reads, index], path-to-stats-dir | null]` in the case of XAM) and extract the
|
||||
* run IDs from the `run_ids` file in the stats directory into the metamap. If the path
|
||||
* to the stats dir is `null`, add an empty list.
|
||||
*
|
||||
@ -58,6 +59,53 @@ def add_run_IDs_to_meta(ch) {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Take a channel of the shape `[meta, reads, path-to-stats-dir | null]` and do the
|
||||
* following:
|
||||
* - For `fastcat`, extract the number of reads from the `n_seqs` file.
|
||||
* - For `bamstats`, extract the number of primary alignments and unmapped reads from
|
||||
* the `bamstats.flagstat.tsv` file.
|
||||
* Then, add add these metrics to the meta map. If the path to the stats dir is `null`,
|
||||
* set the values to 0 when adding them.
|
||||
*
|
||||
* @param ch: input channel of shape `[meta, reads, path-to-stats-dir | null]`
|
||||
* @return: channel with a list of number of reads added to the metamap
|
||||
*/
|
||||
def add_number_of_reads_to_meta(ch, String input_type_format) {
|
||||
// extract reads from fastcat stats / bamstats results and add to metadata
|
||||
ch = ch | map { meta, reads, stats ->
|
||||
// Check that stats directory is present.
|
||||
if (stats) {
|
||||
if (input_type_format == "fastq") {
|
||||
// Stats from fastcat
|
||||
Integer n_seqs = stats.resolve("n_seqs").splitText()[0] as Integer
|
||||
// `meta + [...]` returns a new map which is handy to avoid any
|
||||
// modifying-maps-in-closures weirdness
|
||||
// See https://github.com/nextflow-io/nextflow/issues/2660
|
||||
[meta + [n_seqs: n_seqs], reads, stats]
|
||||
} else {
|
||||
// or bamstats
|
||||
ArrayList stats_csv = stats.resolve("bamstats.flagstat.tsv").splitCsv(header: true, sep:'\t')
|
||||
// get primary alignments and unmapped and sum them
|
||||
Integer n_primary = stats_csv["primary"].collect{it as Integer}.sum()
|
||||
Integer n_unmapped = stats_csv["unmapped"].collect{it as Integer}.sum()
|
||||
// `meta + [...]` returns a new map which is handy to avoid any
|
||||
// modifying-maps-in-closures weirdness
|
||||
// See https://github.com/nextflow-io/nextflow/issues/2660
|
||||
[meta + [n_primary: n_primary, n_unmapped: n_unmapped], reads, stats]
|
||||
}
|
||||
} else {
|
||||
// return defaults if stats is not there
|
||||
if (input_type_format == "fastq") {
|
||||
[meta + [n_seqs: null], reads, stats]
|
||||
} else {
|
||||
[meta + [n_primary: null, n_unmapped: null], reads, stats]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a map of input arguments, find valid FASTQ inputs, and return a channel
|
||||
* with elements of `[metamap, seqs.fastq.gz | null, path-to-fastcat-stats | null]`.
|
||||
@ -86,7 +134,7 @@ def add_run_IDs_to_meta(ch) {
|
||||
def fastq_ingress(Map arguments)
|
||||
{
|
||||
// check arguments
|
||||
Map margs = parse_arguments(arguments, ["fastcat_extra_args": ""])
|
||||
Map margs = parse_arguments("fastq_ingress", arguments, ["fastcat_extra_args": ""])
|
||||
|
||||
ArrayList fq_extensions = [".fastq", ".fastq.gz", ".fq", ".fq.gz"]
|
||||
|
||||
@ -109,7 +157,9 @@ def fastq_ingress(Map arguments)
|
||||
// add sample sheet entries without barcode dirs to the results channel and extract
|
||||
// the run IDs into the metamaps before returning
|
||||
ch_result = ch_result.mix(input.missing.map { [*it, null] })
|
||||
return add_run_IDs_to_meta(ch_result)
|
||||
ch_result_run_IDs = add_run_IDs_to_meta(ch_result)
|
||||
// add number of reads after potential filtering under the field n_seqs
|
||||
return add_number_of_reads_to_meta(ch_result_run_IDs, "fastq")
|
||||
}
|
||||
|
||||
|
||||
@ -118,7 +168,8 @@ def fastq_ingress(Map arguments)
|
||||
* with elements of `[metamap, reads.bam | null, path-to-bamstats-results | null]`.
|
||||
* The second item is `null` for sample sheet entries without a matching barcode
|
||||
* directory or samples containing only uBAM files when `keep_unaligned` is `false`.
|
||||
* The last item is `null` if `bamstats` was not run (it is only run when `stats: true`).
|
||||
* The last item is `null` if `bamstats` was not run (it is only run when `stats:
|
||||
* true`).
|
||||
*
|
||||
* @param arguments: map with arguments containing
|
||||
* - "input": path to either: (i) input (u)BAM file, (ii) top-level directory
|
||||
@ -129,6 +180,9 @@ def fastq_ingress(Map arguments)
|
||||
* - "analyse_unclassified": boolean whether to keep unclassified reads
|
||||
* - "stats": boolean whether to run `bamstats`
|
||||
* - "keep_unaligned": boolean whether to include uBAM files
|
||||
* - "return_fastq": boolean whether to convert to FASTQ (this will always run
|
||||
* `fastcat`)
|
||||
* - "fastcat_extra_args": string with extra arguments to pass to `fastcat`
|
||||
* - "required_sample_types": list of required sample types in the sample sheet
|
||||
* - "watch_path": boolean whether to use `watchPath` and run in streaming mode
|
||||
* @return: channel of `[Map(alias, barcode, type, ...), Path|null, Path|null]`.
|
||||
@ -142,7 +196,11 @@ def fastq_ingress(Map arguments)
|
||||
def xam_ingress(Map arguments)
|
||||
{
|
||||
// check arguments
|
||||
Map margs = parse_arguments(arguments, ["keep_unaligned": false])
|
||||
Map margs = parse_arguments(
|
||||
"xam_ingress",
|
||||
arguments,
|
||||
["keep_unaligned": false, "return_fastq": false, "fastcat_extra_args": ""]
|
||||
)
|
||||
|
||||
// we only accept BAM or uBAM for now (i.e. no SAM or CRAM)
|
||||
ArrayList xam_extensions = [".bam", ".ubam"]
|
||||
@ -153,11 +211,31 @@ def xam_ingress(Map arguments)
|
||||
ch_result = input.dirs
|
||||
| map { meta, path -> [meta, get_target_files_in_dir(path, xam_extensions)] }
|
||||
| mix(input.files)
|
||||
| map{
|
||||
// If there is more than one BAM in each folder we ignore
|
||||
// the indices. For single BAM we add it as a string to the
|
||||
// metadata for later use. If then the BAM returns as position
|
||||
// sorted, the index will be used.
|
||||
meta, paths ->
|
||||
boolean is_array = paths instanceof ArrayList
|
||||
String xai_fn
|
||||
// Using `.uri` or `.Uri()` leads to S3 paths to be prefixed with `s3:///`
|
||||
// instead of `s3://`, causing the workflow to not find the index file.
|
||||
// `.toUriString()` returns the correct path.
|
||||
if (!is_array){
|
||||
def xai = file(paths.toUriString() + ".bai")
|
||||
if (xai.exists()){
|
||||
xai_fn = xai.toUriString()
|
||||
}
|
||||
}
|
||||
[meta + [xai_fn: xai_fn], paths]
|
||||
}
|
||||
| checkBamHeaders
|
||||
| map { meta, paths, is_unaligned_env, mixed_headers_env ->
|
||||
| map { meta, paths, is_unaligned_env, mixed_headers_env, is_sorted_env ->
|
||||
// convert the env. variables from strings ('0' or '1') into bools
|
||||
boolean is_unaligned = is_unaligned_env as int as boolean
|
||||
boolean mixed_headers = mixed_headers_env as int as boolean
|
||||
boolean is_sorted = is_sorted_env as int as boolean
|
||||
// throw an error if there was a sample with mixed headers
|
||||
if (mixed_headers) {
|
||||
error "Found mixed headers in (u)BAM files of sample '${meta.alias}'."
|
||||
@ -165,7 +243,7 @@ def xam_ingress(Map arguments)
|
||||
// add `is_unaligned` to the metamap (note the use of `+` to create a copy of
|
||||
// `meta` to avoid modifying every item in the channel;
|
||||
// https://github.com/nextflow-io/nextflow/issues/2660)
|
||||
[meta + [is_unaligned: is_unaligned], paths]
|
||||
[meta + [is_unaligned: is_unaligned, is_sorted: is_sorted], paths]
|
||||
}
|
||||
| branch { meta, paths ->
|
||||
// set `paths` to `null` for uBAM samples if unallowed (they will be added to
|
||||
@ -176,13 +254,12 @@ def xam_ingress(Map arguments)
|
||||
}
|
||||
// get the number of files (`paths` can be a list, a single path, or `null`)
|
||||
int n_files = paths instanceof List ? paths.size() : (paths ? 1 : 0)
|
||||
// Preparations finished; we can do the branching now. There will be 3 branches
|
||||
// Preparations finished; we can do the branching now. There will be 5 branches
|
||||
// depending on the number of files per sample and whether the reads are already
|
||||
// aligned:
|
||||
// * no_op_needed: no need to do anything; just add to the final results channel
|
||||
// downstream
|
||||
// - no files
|
||||
// - a single unaligned file
|
||||
// * no files: no need to do anything
|
||||
// * indexed: a single sorted and indexed BAM file. Index will be validated.
|
||||
// * to_index: a single sorted, but not indexed, BAM file
|
||||
// * to_catsort: `samtools cat` into `samtools sort`
|
||||
// - a single aligned file
|
||||
// - more than one unaligned file
|
||||
@ -191,12 +268,36 @@ def xam_ingress(Map arguments)
|
||||
// open file descriptors)
|
||||
// * to_merge: flatMap > sort > group > merge
|
||||
// - between 1 and `N_OPEN_FILES_LIMIT` aligned files
|
||||
no_op_needed: (n_files == 0) || (n_files == 1 && meta["is_unaligned"])
|
||||
no_files: n_files == 0
|
||||
indexed: \
|
||||
n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && meta["xai_fn"]
|
||||
to_index:
|
||||
n_files == 1 && (meta["is_unaligned"] || meta["is_sorted"]) && !meta["xai_fn"]
|
||||
to_catsort: \
|
||||
(n_files == 1) || (n_files > N_OPEN_FILES_LIMIT) || meta["is_unaligned"]
|
||||
to_merge: true
|
||||
}
|
||||
|
||||
if (margs["return_fastq"]) {
|
||||
// only run samtools fastq on samples with at least one file
|
||||
ch_to_fastq = ch_result.indexed.mix(
|
||||
ch_result.to_index,
|
||||
ch_result.to_merge,
|
||||
ch_result.to_catsort
|
||||
)
|
||||
|
||||
// input.missing: sample sheet entries without barcode dirs
|
||||
ch_result = input.missing
|
||||
| mix(ch_result.no_files)
|
||||
| map { [*it, null] }
|
||||
| mix(bamToFastq(ch_to_fastq, margs["fastcat_extra_args"]))
|
||||
| map{
|
||||
meta, path, stats ->
|
||||
[meta.findAll { it.key !in ['xai_fn', 'is_sorted'] }, path, stats]
|
||||
}
|
||||
return add_number_of_reads_to_meta(add_run_IDs_to_meta(ch_result), "fastq")
|
||||
}
|
||||
|
||||
// deal with samples with few-enough files for `samtools merge` first
|
||||
ch_merged = ch_result.to_merge
|
||||
| flatMap { meta, paths -> paths.collect { [meta, it] } }
|
||||
@ -208,9 +309,59 @@ def xam_ingress(Map arguments)
|
||||
ch_catsorted = ch_result.to_catsort
|
||||
| catSortBams
|
||||
|
||||
ch_result = input.missing
|
||||
// Validate the index of the input BAM.
|
||||
// If the input BAM index is invalid, regenerate it.
|
||||
// First separate the BAM from the null input channels.
|
||||
ch_to_validate = ch_result.indexed
|
||||
| map{
|
||||
meta, paths ->
|
||||
bai = paths && meta.xai_fn ? file(meta.xai_fn) : null
|
||||
[meta, paths, bai]
|
||||
}
|
||||
| branch {
|
||||
meta, paths, bai ->
|
||||
to_validate: paths && bai
|
||||
no_op_needed: true
|
||||
}
|
||||
|
||||
// Validate non-null files with index
|
||||
ch_validated = ch_to_validate.to_validate
|
||||
| validateIndex
|
||||
| branch {
|
||||
meta, bam, bai, has_valid_index_env ->
|
||||
boolean has_valid_index = has_valid_index_env as int as boolean
|
||||
// Split if it is a valid index
|
||||
valid_idx: has_valid_index
|
||||
return [meta, bam, bai]
|
||||
invalid_idx: true
|
||||
return [meta, bam]
|
||||
}
|
||||
|
||||
// Create channel for no_op needed (null channels and valid indexes)
|
||||
ch_no_op = ch_validated.valid_idx
|
||||
| mix(ch_to_validate.no_op_needed)
|
||||
|
||||
// Re-index sorted-not-indexed BAM file
|
||||
ch_indexed = ch_result.to_index
|
||||
| mix( ch_validated.invalid_idx )
|
||||
| samtools_index
|
||||
|
||||
// Add extra null for the missing index to input.missing
|
||||
// as well as the missing metadata.
|
||||
// input.missing: sample sheet entries without barcode dirs
|
||||
ch_missing = input.missing
|
||||
| mix(
|
||||
ch_result.no_op_needed,
|
||||
ch_result.no_files,
|
||||
)
|
||||
| map{
|
||||
meta, paths ->
|
||||
[meta + [xai_fn: null, is_sorted: false], paths, null]
|
||||
}
|
||||
|
||||
// Combine all possible inputs
|
||||
ch_result = ch_missing | mix(
|
||||
ch_no_op,
|
||||
ch_indexed,
|
||||
ch_merged,
|
||||
ch_catsorted,
|
||||
)
|
||||
@ -218,19 +369,97 @@ def xam_ingress(Map arguments)
|
||||
// run `bamstats` if requested
|
||||
if (margs["stats"]) {
|
||||
// branch and run `bamstats` only on the non-`null` paths
|
||||
ch_result = ch_result.branch { meta, path ->
|
||||
ch_result = ch_result.branch { meta, path, index ->
|
||||
has_reads: path
|
||||
is_null: true
|
||||
}
|
||||
ch_bamstats = bamstats(ch_result.has_reads)
|
||||
ch_result = add_run_IDs_to_meta(ch_bamstats) | mix(ch_result.is_null)
|
||||
|
||||
// the channel comes from xam_ingress also have the BAM index in it.
|
||||
// Handle this by placing them in a nested array, maintaining the structure
|
||||
// from fastq_ingress. We do not use variable name as assigning variable
|
||||
// name with a tuple not matching (e.g. meta, bam, bai, stats <- [meta, bam, stats] )
|
||||
// causes the workflow to crash.
|
||||
ch_result = ch_bamstats
|
||||
| map{
|
||||
it[3] ? [it[0], [it[1], it[2]], it[3]] : it
|
||||
}
|
||||
| add_run_IDs_to_meta
|
||||
| map{
|
||||
it.flatten()
|
||||
}
|
||||
| mix(
|
||||
ch_result.is_null.map{it + [null]}
|
||||
)
|
||||
} else {
|
||||
// add `null` instead of path to `bamstats` results dir
|
||||
ch_result = ch_result | map { meta, bam -> [meta, bam, null] }
|
||||
ch_result = ch_result | map { meta, bam, bai -> [meta, bam, bai, null] }
|
||||
}
|
||||
|
||||
// Remove metadata that are unnecessary downstream:
|
||||
// meta.xai_fn: not needed, as it will be part of the channel as a file
|
||||
// meta.is_sorted: if data are aligned, they will also be sorted/indexed
|
||||
//
|
||||
// The output meta can contain the following flags:
|
||||
// [
|
||||
// barcode: always present
|
||||
// type: always present
|
||||
// run_id: always present, but can be empty (i.e. `[]`)
|
||||
// alias: always present
|
||||
// n_primary: always present, but can be `null`
|
||||
// n_unmapped: always present, but can be `null`
|
||||
// is_unaligned: present if there is a (u)BAM file
|
||||
// ]
|
||||
ch_result = add_number_of_reads_to_meta(
|
||||
ch_result
|
||||
| map{
|
||||
meta, bam, bai, stats ->
|
||||
[meta.findAll { it.key !in ['xai_fn', 'is_sorted'] }, [bam, bai], stats]
|
||||
},
|
||||
"xam"
|
||||
)
|
||||
| map{
|
||||
it.flatten()
|
||||
}
|
||||
|
||||
return ch_result
|
||||
}
|
||||
|
||||
process bamToFastq {
|
||||
label "ingress"
|
||||
label "wf_common"
|
||||
cpus 4
|
||||
memory "2 GB"
|
||||
input:
|
||||
tuple val(meta), path(bams, stageAs: "input_dir/reads*.bam")
|
||||
val extra_args
|
||||
output: tuple val(meta), path("seqs.fastq.gz"), path("fastcat_stats")
|
||||
script:
|
||||
"""
|
||||
mkdir fastcat_stats
|
||||
|
||||
# Save file as compressed fastq
|
||||
fastcat \
|
||||
-s ${meta["alias"]} \
|
||||
-r >(bgzip -c > fastcat_stats/per-read-stats.tsv.gz) \
|
||||
-f fastcat_stats/per-file-stats.tsv \
|
||||
--histograms histograms \
|
||||
$extra_args \
|
||||
<(
|
||||
samtools cat -b <(find input_dir -name 'reads*.bam') | \
|
||||
samtools fastq - -n -T '*' -o - -0 -
|
||||
) \
|
||||
| bgzip -c > seqs.fastq.gz
|
||||
|
||||
mv histograms/* fastcat_stats
|
||||
|
||||
# extract the run IDs and number of sequences (n_seqs) from the per-read stats
|
||||
csvtk freq -tf runid fastcat_stats/per-read-stats.tsv.gz \
|
||||
| csvtk del-header \
|
||||
| tee >(cut -f 1 | sort > "fastcat_stats/run_ids") \
|
||||
| awk 'BEGIN{n=0}; {n+=\$2}; END{print n}' > "fastcat_stats/n_seqs"
|
||||
"""
|
||||
}
|
||||
|
||||
process checkBamHeaders {
|
||||
label "ingress"
|
||||
@ -246,6 +475,7 @@ process checkBamHeaders {
|
||||
path("input_dir/reads*.bam", includeInputs: true),
|
||||
env(IS_UNALIGNED),
|
||||
env(MIXED_HEADERS),
|
||||
env(IS_SORTED),
|
||||
)
|
||||
script:
|
||||
"""
|
||||
@ -255,17 +485,41 @@ process checkBamHeaders {
|
||||
}
|
||||
|
||||
|
||||
process validateIndex {
|
||||
label "ingress"
|
||||
label "wf_common"
|
||||
cpus 1
|
||||
memory "2 GB"
|
||||
input: tuple val(meta), path("reads.bam"), path("reads.bam.bai")
|
||||
output:
|
||||
// set the two env variables by `eval`-ing the output of the python script
|
||||
// checking the XAM headers
|
||||
tuple(
|
||||
val(meta),
|
||||
path("reads.bam", includeInputs: true),
|
||||
path("reads.bam.bai", includeInputs: true),
|
||||
env(HAS_VALID_INDEX)
|
||||
)
|
||||
script:
|
||||
"""
|
||||
workflow-glue check_xam_index reads.bam > env.vars
|
||||
source env.vars
|
||||
"""
|
||||
}
|
||||
|
||||
|
||||
process mergeBams {
|
||||
label "ingress"
|
||||
label "wf_common"
|
||||
cpus 3
|
||||
memory "4 GB"
|
||||
input: tuple val(meta), path("input_bams/reads*.bam")
|
||||
output: tuple val(meta), path("reads.bam")
|
||||
shell:
|
||||
input: tuple val(meta), path("input_bams/reads*.bam"), path("input_bams/reads*.bam.bai")
|
||||
output: tuple val(meta), path("reads.bam"), path("reads.bam.bai")
|
||||
script:
|
||||
def merge_threads = Math.max(1, task.cpus - 1)
|
||||
"""
|
||||
samtools merge -@ ${task.cpus - 1} \
|
||||
-b <(find input_bams -name 'reads*.bam') -o reads.bam
|
||||
samtools merge -@ ${merge_threads} \
|
||||
-b <(find input_bams -name 'reads*.bam') --write-index -o reads.bam##idx##reads.bam.bai
|
||||
"""
|
||||
}
|
||||
|
||||
@ -276,11 +530,12 @@ process catSortBams {
|
||||
cpus 4
|
||||
memory "4 GB"
|
||||
input: tuple val(meta), path("input_bams/reads*.bam")
|
||||
output: tuple val(meta), path("reads.bam")
|
||||
output: tuple val(meta), path("reads.bam"), path("reads.bam.bai")
|
||||
script:
|
||||
def sort_threads = Math.max(1, task.cpus - 2)
|
||||
"""
|
||||
samtools cat -b <(find input_bams -name 'reads*.bam') \
|
||||
| samtools sort - -@ ${task.cpus - 2} -o reads.bam
|
||||
| samtools sort - -@ ${sort_threads} --write-index -o reads.bam##idx##reads.bam.bai
|
||||
"""
|
||||
}
|
||||
|
||||
@ -291,10 +546,11 @@ process sortBam {
|
||||
cpus 3
|
||||
memory "4 GB"
|
||||
input: tuple val(meta), path("reads.bam")
|
||||
output: tuple val(meta), path("reads.sorted.bam")
|
||||
output: tuple val(meta), path("reads.sorted.bam"), path("reads.sorted.bam.bai")
|
||||
script:
|
||||
def sort_threads = Math.max(1, task.cpus - 1)
|
||||
"""
|
||||
samtools sort -@ ${task.cpus - 1} reads.bam -o reads.sorted.bam
|
||||
samtools sort --write-index -@ ${sort_threads} reads.bam -o reads.sorted.bam##idx##reads.sorted.bam.bai
|
||||
"""
|
||||
}
|
||||
|
||||
@ -305,10 +561,11 @@ process bamstats {
|
||||
cpus 3
|
||||
memory "4 GB"
|
||||
input:
|
||||
tuple val(meta), path("reads.bam")
|
||||
tuple val(meta), path("reads.bam"), path("reads.bam.bai")
|
||||
output:
|
||||
tuple val(meta),
|
||||
path("reads.bam"),
|
||||
path("reads.bam.bai"),
|
||||
path("bamstats_results")
|
||||
script:
|
||||
def bamstats_threads = Math.max(1, task.cpus - 1)
|
||||
@ -325,8 +582,6 @@ process bamstats {
|
||||
| csvtk del-header | sort | uniq > bamstats_results/run_ids
|
||||
"""
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Run `watchPath` on the input directory and return a channel of shape [metamap,
|
||||
* path-to-target-file]. The meta data is taken from the sample sheet in case one was
|
||||
@ -473,9 +728,11 @@ process fastcat {
|
||||
| bgzip > $out
|
||||
|
||||
mv histograms/* $fastcat_stats_outdir
|
||||
# extract the run IDs from the per-read stats
|
||||
csvtk cut -tf runid $fastcat_stats_outdir/per-read-stats.tsv.gz \
|
||||
| csvtk del-header | sort | uniq > $fastcat_stats_outdir/run_ids
|
||||
# extract the run IDs and number of sequences (n_seqs) from the per-read stats
|
||||
csvtk freq -tf runid $fastcat_stats_outdir/per-read-stats.tsv.gz \
|
||||
| csvtk del-header \
|
||||
| tee >(cut -f 1 | sort > "$fastcat_stats_outdir/run_ids") \
|
||||
| awk 'BEGIN{n=0}; {n+=\$2}; END{print n}' > "$fastcat_stats_outdir/n_seqs"
|
||||
"""
|
||||
}
|
||||
|
||||
@ -489,7 +746,7 @@ process fastcat {
|
||||
* the argument-parsing to be tailored to a particular ingress function)
|
||||
* @return: map of parsed arguments
|
||||
*/
|
||||
Map parse_arguments(Map arguments, Map extra_kwargs=[:]) {
|
||||
Map parse_arguments(String func_name, Map arguments, Map extra_kwargs=[:]) {
|
||||
ArrayList required_args = ["input"]
|
||||
Map default_kwargs = [
|
||||
"sample": null,
|
||||
@ -502,7 +759,7 @@ Map parse_arguments(Map arguments, Map extra_kwargs=[:]) {
|
||||
ArgumentParser parser = new ArgumentParser(
|
||||
args: required_args,
|
||||
kwargs: default_kwargs + extra_kwargs,
|
||||
name: "fastq_ingress")
|
||||
name: func_name)
|
||||
return parser.parse_args(arguments)
|
||||
}
|
||||
|
||||
@ -726,10 +983,10 @@ def get_sample_sheet(Path sample_sheet, ArrayList required_sample_types) {
|
||||
// in STDOUT. Thus, we use the somewhat clunky construct with `concat` and `last`
|
||||
// below. This lets the CSV channel only start to emit once the error checking is
|
||||
// done.
|
||||
ch_err = validate_sample_sheet(sample_sheet, required_sample_types).map {
|
||||
ch_err = validate_sample_sheet(sample_sheet, required_sample_types).map { stdoutput, sample_sheet_file ->
|
||||
// check if there was an error message
|
||||
if (it) error "Invalid sample sheet: ${it}."
|
||||
it
|
||||
if (stdoutput) error "Invalid sample sheet: ${stdoutput}."
|
||||
stdoutput
|
||||
}
|
||||
// concat the channel holding the path to the sample sheet to `ch_err` and call
|
||||
// `.last()` to make sure that the error-checking closure above executes before
|
||||
@ -743,13 +1000,14 @@ def get_sample_sheet(Path sample_sheet, ArrayList required_sample_types) {
|
||||
/**
|
||||
* Python script for validating a sample sheet. The script will write messages
|
||||
* to STDOUT if the sample sheet is invalid. In case there are no issues, no
|
||||
* message is emitted.
|
||||
* message is emitted. The sample sheet will be published to the output dir.
|
||||
*
|
||||
* @param: path to sample sheet CSV
|
||||
* @param: list of required sample types (optional)
|
||||
* @return: string (optional)
|
||||
*/
|
||||
process validate_sample_sheet {
|
||||
publishDir params.out_dir, mode: 'copy', overwrite: true
|
||||
cpus 1
|
||||
label "ingress"
|
||||
label "wf_common"
|
||||
@ -757,10 +1015,27 @@ process validate_sample_sheet {
|
||||
input:
|
||||
path "sample_sheet.csv"
|
||||
val required_sample_types
|
||||
output: stdout
|
||||
output:
|
||||
tuple stdout, path("sample_sheet.csv")
|
||||
script:
|
||||
String req_types_arg = required_sample_types ? "--required_sample_types "+required_sample_types.join(" ") : ""
|
||||
"""
|
||||
workflow-glue check_sample_sheet sample_sheet.csv $req_types_arg
|
||||
"""
|
||||
}
|
||||
|
||||
// Generate an index for an input XAM file
|
||||
process samtools_index {
|
||||
cpus 4
|
||||
label "ingress"
|
||||
label "wf_common"
|
||||
memory 4.GB
|
||||
input:
|
||||
tuple val(meta), path("reads.bam")
|
||||
output:
|
||||
tuple val(meta), path("reads.bam"), path("reads.bam.bai")
|
||||
script:
|
||||
"""
|
||||
samtools index -@ $task.cpus reads.bam
|
||||
"""
|
||||
}
|
||||
|
||||
@ -105,7 +105,7 @@ params {
|
||||
]
|
||||
agent = null
|
||||
container_sha = "shae7c9f184996a384e99be68e790f0612f0c732867"
|
||||
common_sha = "sha1c5febff9f75143710826498b093d9769a5edbb9"
|
||||
common_sha = "sha645176f98b8780851f9c476a064d44c2ae76ddf6"
|
||||
}
|
||||
}
|
||||
|
||||
@ -204,4 +204,5 @@ trace {
|
||||
|
||||
env {
|
||||
PYTHONNOUSERSITE = 1
|
||||
JAVA_TOOL_OPTIONS = "-Xlog:disable -Xlog:all=warning:stderr"
|
||||
}
|
||||
|
||||
@ -13,6 +13,9 @@ process jaffal{
|
||||
tuple val(sample_id), path("jaffal_output_$sample_id/*jaffa_results.csv"), emit: results_csv
|
||||
script:
|
||||
"""
|
||||
# Jaffa requires a virtual environment
|
||||
# so override JAVA_TOOL_OPTIONS env variable.
|
||||
JAVA_TOOL_OPTIONS=""
|
||||
JAFFAOUT=jaffal_output_$sample_id
|
||||
|
||||
# JAFFAL exists with status code 1 when there's 0 fusion hits. Prevent this with '||:'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user