226 lines
16 KiB
YAML
226 lines
16 KiB
YAML
# Include shared CI
|
|
include:
|
|
- project: "epi2melabs/ci-templates"
|
|
file: "wf-containers.yaml"
|
|
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch"
|
|
CI_FLAVOUR: "new"
|
|
PYTEST_CONTAINER_NAME: "wf-common"
|
|
PYTEST_CONTAINER_CONFIG_KEY: "common_sha"
|
|
PYTEST_TESTS_PATH: "bin/workflow_glue/tests/common"
|
|
RTEST_CONTAINER_NAME: "wf-transcriptomes-core"
|
|
RTEST_CONTAINER_CONFIG_KEY: "container_sha"
|
|
WF_TEMPLATE_ENFORCEMENT_REF: "v6.1.0-rc2"
|
|
EKS_RUNNER_SIZE: "xlarge-highio"
|
|
|
|
pytest_wfcontainer:
|
|
extends: pytest
|
|
variables:
|
|
PYTEST_CONTAINER_NAME: "wf-transcriptomes-core"
|
|
PYTEST_CONTAINER_CONFIG_KEY: "container_sha"
|
|
PYTEST_TESTS_PATH: "bin/workflow_glue/tests/wf"
|
|
|
|
macos-run:
|
|
# Let's avoid those ARM64 runners for now
|
|
tags:
|
|
- macos
|
|
- x86
|
|
|
|
aws-run:
|
|
rules:
|
|
- when: never
|
|
|
|
docker-run:
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- ${CI_PROJECT_NAME}
|
|
- .nextflow.log
|
|
exclude:
|
|
- ${CI_PROJECT_NAME}/**/*.gtf
|
|
- ${CI_PROJECT_NAME}/**/*.gtf.gz
|
|
- ${CI_PROJECT_NAME}/**/*.gff3
|
|
- ${CI_PROJECT_NAME}/**/*.gff3.gz
|
|
- ${CI_PROJECT_NAME}/**/*.gff
|
|
- ${CI_PROJECT_NAME}/**/*.gff.gz
|
|
- ${CI_PROJECT_NAME}/**/*.fna
|
|
- ${CI_PROJECT_NAME}/**/*.fasta
|
|
- ${CI_PROJECT_NAME}/**/*.mmi
|
|
- ${CI_PROJECT_NAME}/data/**/*
|
|
- ${CI_PROJECT_NAME}/store_dir/**/*
|
|
|
|
|
|
# Define a 1D job matrix to inject a variable named MATRIX_NAME into
|
|
# the CI environment, we can use the value of MATRIX_NAME to determine
|
|
# which options to apply as part of the rules block below
|
|
# NOTE There is a slightly cleaner way to define this matrix to include
|
|
# the variables, but it is broken when using long strings! See CW-756
|
|
parallel:
|
|
matrix:
|
|
- MATRIX_NAME: [
|
|
"int_discover_dna", "int_fixed_rna", "int_de_control_vs_control",
|
|
"smoke_discover", "smoke_fixed", "smoke_direct_rna", "smoke_de",
|
|
"mouse_de_0countquant", "mods_bigwig_igv", "mismatch-sample-alias",
|
|
"mouse_splice_error", "mouse_splice_realign"
|
|
]
|
|
rules:
|
|
# NOTE As we're overriding the rules block for the included docker-run
|
|
# we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run
|
|
# being incorrectly scheduled for "detached merge request pipelines" etc.
|
|
# Guardrail: never schedule docker-run on detached/non-standard branch context.
|
|
- if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template")
|
|
when: never
|
|
|
|
# Integration: larger discover-mode run on representative cDNA test bundle.
|
|
- if: $MATRIX_NAME == "int_discover_dna"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf \
|
|
--transcriptome_mode discover"
|
|
NF_IGNORE_PROCESSES: filter_unstranded_annotation,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
|
|
|
# Integration: fixed-annotation + direct-RNA + DE + IGV with GFF input on richer dataset.
|
|
- if: $MATRIX_NAME == "int_fixed_rna"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \
|
|
--sample_sheet test_data/sample_sheet.csv \
|
|
--de_analysis \
|
|
--direct_rna \
|
|
--transcriptome_mode fixed_annotation \
|
|
--igv \
|
|
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config "
|
|
NF_IGNORE_PROCESSES: >
|
|
gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
|
|
build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
|
|
|
# Integration: demo-data control-vs-control to test pathological DE/DTU failure modes and graceful failure handling.
|
|
# See also unit tests in test_de_analysis.R
|
|
- if: $MATRIX_NAME == "int_de_control_vs_control"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "\
|
|
mkdir -p ${CI_PROJECT_NAME}/data/ \
|
|
&& echo 'Downlading demo data bundle and config' \
|
|
&& wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz \
|
|
&& tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ \
|
|
&& wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config \
|
|
&& echo 'Simulating control-vs-control by copying each sample to a replicate with identical reads' \
|
|
&& rm -rf ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \
|
|
&& cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode01 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 \
|
|
&& cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode02 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 \
|
|
&& cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode03 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \
|
|
&& echo 'Finished data preparation step' \
|
|
"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
|
--de_analysis \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
|
|
--direct_rna \
|
|
--sample_sheet test_data/sample_sheet.csv \
|
|
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json &&
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv &&
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv &&
|
|
jq -e '.contrasts["condition_treated_vs_control"].dge_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null &&
|
|
jq -e '.contrasts["condition_treated_vs_control"].dtu_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null
|
|
|
|
# Smoke: quick discover-mode sanity check for core cohort outputs.
|
|
- if: $MATRIX_NAME == "smoke_discover"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf &&
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/sampleA.transcriptome.fa
|
|
|
|
# Smoke: fixed-annotation path sanity check for quantification outputs.
|
|
- if: $MATRIX_NAME == "smoke_fixed"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --transcriptome_mode fixed_annotation"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf &&
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/transcript_counts.tsv
|
|
|
|
# Smoke: direct-RNA alignment profile and downstream SQANTI output presence.
|
|
- if: $MATRIX_NAME == "smoke_direct_rna"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --direct_rna"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/alignment/reads.bam &&
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/sqanti/classification_summary.tsv
|
|
|
|
# Smoke: end-to-end DE/DTU wiring and expected contrast output files.
|
|
- if: $MATRIX_NAME == "smoke_de"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv &&
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv &&
|
|
[ "$(find ${CI_PROJECT_NAME}/samples -type f -name 'gene_counts.tsv' | wc -l)" -eq 4 ]
|
|
- if: $MATRIX_NAME == "mismatch-sample-alias"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de/barcode01/reads.fastq --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf"
|
|
ASSERT_NEXTFLOW_FAILURE: "1"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
grep -F "Sample alias 'reads' was not found in the sample_sheet alias column." .nextflow.log
|
|
# Tests a common error when there are 0 annotation counts for a chunk
|
|
- if: $MATRIX_NAME == "mouse_de_0countquant"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_subset_test.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz -C ${CI_PROJECT_NAME}/data/"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/mouse_subset_test/samples \
|
|
--de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.gtf.gz \
|
|
--direct_rna --sample_sheet ${CI_PROJECT_NAME}/data/mouse_subset_test/sample_sheet.csv"
|
|
# MM/ML tag test
|
|
- if: $MATRIX_NAME == "mods_bigwig_igv"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mods_rna_subset.tar.gz -O ${CI_PROJECT_NAME}/data/mods_rna_subset.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mods_rna_subset.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/wf-transcriptomes-demo/gencode.v22.annotation.chr20.gtf -O ${CI_PROJECT_NAME}/data/gencode.v22.annotation.chr20.gtf && wget -nv https://ont-open-data.s3.amazonaws.com/references/human/GRCh38/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz -O ${CI_PROJECT_NAME}/data/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz"
|
|
NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/mods_rna_subset/ \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/gencode.v22.annotation.chr20.gtf \
|
|
--sample_sheet ${CI_PROJECT_NAME}/data/mods_rna_subset/sample_sheet.csv \
|
|
--igv"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
for sample in sample01 sample02; do
|
|
for suffix in mods.bedmethyl.gz mods.inosine.bw mods.m5C.bw mods.m6A.bw mods.pseU.bw mods.summary.tsv; do
|
|
test -f ${CI_PROJECT_NAME}/samples/$${sample}/mods/$${sample}.$${suffix};
|
|
done;
|
|
done;
|
|
|
|
- if: $MATRIX_NAME == "mouse_splice_error"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_splice_fail.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_splice_fail.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_splice_fail.tar.gz -C ${CI_PROJECT_NAME}/data/"
|
|
NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/mouse_splice_fail/samples \
|
|
--de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_splice_fail/mouse_subset.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/mouse_splice_fail/mouse_subset.gtf.gz \
|
|
--sample_sheet ${CI_PROJECT_NAME}/data/mouse_splice_fail/sample_sheet.csv"
|
|
ASSERT_NEXTFLOW_FAILURE: "1"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
grep -qF 'Cannot proceed with mixed splice-aware CIGAR evidence in input BAMs.' .nextflow.log
|
|
- if: $MATRIX_NAME == "mouse_splice_realign"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_splice_realign.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_splice_realign.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_splice_realign.tar.gz -C ${CI_PROJECT_NAME}/data/"
|
|
NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/mouse_splice_realign/samples \
|
|
--de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_splice_realign/mouse_subset.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/mouse_splice_realign/mouse_subset.gtf.gz \
|
|
--sample_sheet ${CI_PROJECT_NAME}/data/mouse_splice_realign/sample_sheet.csv"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
grep -q 'bam_alignment' ${CI_PROJECT_NAME}/execution/trace.txt && grep -q 'No input BAMs appear to contain splice-aware CIGAR evidence. The workflow will realign all inputs.' .nextflow.log
|
|
|
|
|
|
|
|
singularity-run:
|
|
rules:
|
|
- when: never
|