# Include shared CI include: - project: "epi2melabs/ci-templates" file: "wf-containers.yaml" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch" CI_FLAVOUR: "new" PYTEST_CONTAINER_NAME: "wf-common" PYTEST_CONTAINER_CONFIG_KEY: "common_sha" PYTEST_TESTS_PATH: "bin/workflow_glue/tests/common" RTEST_CONTAINER_NAME: "wf-transcriptomes-core" RTEST_CONTAINER_CONFIG_KEY: "container_sha" WF_TEMPLATE_ENFORCEMENT_REF: "v6.1.0-rc2" EKS_RUNNER_SIZE: "xlarge-highio" pytest_wfcontainer: extends: pytest variables: PYTEST_CONTAINER_NAME: "wf-transcriptomes-core" PYTEST_CONTAINER_CONFIG_KEY: "container_sha" PYTEST_TESTS_PATH: "bin/workflow_glue/tests/wf" macos-run: # Let's avoid those ARM64 runners for now tags: - macos - x86 aws-run: rules: - when: never docker-run: artifacts: when: always paths: - ${CI_PROJECT_NAME} - .nextflow.log exclude: - ${CI_PROJECT_NAME}/**/*.gtf - ${CI_PROJECT_NAME}/**/*.gtf.gz - ${CI_PROJECT_NAME}/**/*.gff3 - ${CI_PROJECT_NAME}/**/*.gff3.gz - ${CI_PROJECT_NAME}/**/*.gff - ${CI_PROJECT_NAME}/**/*.gff.gz - ${CI_PROJECT_NAME}/**/*.fna - ${CI_PROJECT_NAME}/**/*.fasta - ${CI_PROJECT_NAME}/**/*.mmi - ${CI_PROJECT_NAME}/data/**/* - ${CI_PROJECT_NAME}/store_dir/**/* # Define a 1D job matrix to inject a variable named MATRIX_NAME into # the CI environment, we can use the value of MATRIX_NAME to determine # which options to apply as part of the rules block below # NOTE There is a slightly cleaner way to define this matrix to include # the variables, but it is broken when using long strings! See CW-756 parallel: matrix: - MATRIX_NAME: [ "int_discover_dna", "int_fixed_rna", "int_de_control_vs_control", "smoke_discover", "smoke_fixed", "smoke_direct_rna", "smoke_de", "mouse_de_0countquant", "mods_bigwig_igv", "mismatch-sample-alias", "mouse_splice_error", "mouse_splice_realign" ] rules: # NOTE As we're overriding the rules block for the included docker-run # we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run # being incorrectly scheduled for "detached merge request pipelines" etc. # Guardrail: never schedule docker-run on detached/non-standard branch context. - if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template") when: never # Integration: larger discover-mode run on representative cDNA test bundle. - if: $MATRIX_NAME == "int_discover_dna" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf \ --transcriptome_mode discover" NF_IGNORE_PROCESSES: filter_unstranded_annotation,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome # Integration: fixed-annotation + direct-RNA + DE + IGV with GFF input on richer dataset. - if: $MATRIX_NAME == "int_fixed_rna" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \ --sample_sheet test_data/sample_sheet.csv \ --de_analysis \ --direct_rna \ --transcriptome_mode fixed_annotation \ --igv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config " NF_IGNORE_PROCESSES: > gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome # Integration: demo-data control-vs-control to test pathological DE/DTU failure modes and graceful failure handling. # See also unit tests in test_de_analysis.R - if: $MATRIX_NAME == "int_de_control_vs_control" variables: NF_BEFORE_SCRIPT: "\ mkdir -p ${CI_PROJECT_NAME}/data/ \ && echo 'Downlading demo data bundle and config' \ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz \ && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ \ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config \ && echo 'Simulating control-vs-control by copying each sample to a replicate with identical reads' \ && rm -rf ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \ && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode01 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 \ && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode02 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 \ && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode03 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \ && echo 'Finished data preparation step' \ " NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna \ --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json && test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv && test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv && jq -e '.contrasts["condition_treated_vs_control"].dge_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null && jq -e '.contrasts["condition_treated_vs_control"].dtu_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null # Smoke: quick discover-mode sanity check for core cohort outputs. - if: $MATRIX_NAME == "smoke_discover" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf && test -f ${CI_PROJECT_NAME}/samples/sampleA/sampleA.transcriptome.fa # Smoke: fixed-annotation path sanity check for quantification outputs. - if: $MATRIX_NAME == "smoke_fixed" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --transcriptome_mode fixed_annotation" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf && test -f ${CI_PROJECT_NAME}/samples/sampleA/transcript_counts.tsv # Smoke: direct-RNA alignment profile and downstream SQANTI output presence. - if: $MATRIX_NAME == "smoke_direct_rna" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --direct_rna" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/samples/sampleA/alignment/reads.bam && test -f ${CI_PROJECT_NAME}/samples/sampleA/sqanti/classification_summary.tsv # Smoke: end-to-end DE/DTU wiring and expected contrast output files. - if: $MATRIX_NAME == "smoke_de" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv && test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv && [ "$(find ${CI_PROJECT_NAME}/samples -type f -name 'gene_counts.tsv' | wc -l)" -eq 4 ] - if: $MATRIX_NAME == "mismatch-sample-alias" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de/barcode01/reads.fastq --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf" ASSERT_NEXTFLOW_FAILURE: "1" AFTER_NEXTFLOW_CMD: > grep -F "Sample alias 'reads' was not found in the sample_sheet alias column." .nextflow.log # Tests a common error when there are 0 annotation counts for a chunk - if: $MATRIX_NAME == "mouse_de_0countquant" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_subset_test.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz -C ${CI_PROJECT_NAME}/data/" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/mouse_subset_test/samples \ --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.gtf.gz \ --direct_rna --sample_sheet ${CI_PROJECT_NAME}/data/mouse_subset_test/sample_sheet.csv" # MM/ML tag test - if: $MATRIX_NAME == "mods_bigwig_igv" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mods_rna_subset.tar.gz -O ${CI_PROJECT_NAME}/data/mods_rna_subset.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mods_rna_subset.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/wf-transcriptomes-demo/gencode.v22.annotation.chr20.gtf -O ${CI_PROJECT_NAME}/data/gencode.v22.annotation.chr20.gtf && wget -nv https://ont-open-data.s3.amazonaws.com/references/human/GRCh38/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz -O ${CI_PROJECT_NAME}/data/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz" NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/mods_rna_subset/ \ --ref_genome ${CI_PROJECT_NAME}/data/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz \ --ref_annotation ${CI_PROJECT_NAME}/data/gencode.v22.annotation.chr20.gtf \ --sample_sheet ${CI_PROJECT_NAME}/data/mods_rna_subset/sample_sheet.csv \ --igv" AFTER_NEXTFLOW_CMD: > for sample in sample01 sample02; do for suffix in mods.bedmethyl.gz mods.inosine.bw mods.m5C.bw mods.m6A.bw mods.pseU.bw mods.summary.tsv; do test -f ${CI_PROJECT_NAME}/samples/$${sample}/mods/$${sample}.$${suffix}; done; done; - if: $MATRIX_NAME == "mouse_splice_error" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_splice_fail.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_splice_fail.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_splice_fail.tar.gz -C ${CI_PROJECT_NAME}/data/" NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/mouse_splice_fail/samples \ --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_splice_fail/mouse_subset.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/mouse_splice_fail/mouse_subset.gtf.gz \ --sample_sheet ${CI_PROJECT_NAME}/data/mouse_splice_fail/sample_sheet.csv" ASSERT_NEXTFLOW_FAILURE: "1" AFTER_NEXTFLOW_CMD: > grep -qF 'Cannot proceed with mixed splice-aware CIGAR evidence in input BAMs.' .nextflow.log - if: $MATRIX_NAME == "mouse_splice_realign" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_splice_realign.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_splice_realign.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_splice_realign.tar.gz -C ${CI_PROJECT_NAME}/data/" NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/mouse_splice_realign/samples \ --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_splice_realign/mouse_subset.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/mouse_splice_realign/mouse_subset.gtf.gz \ --sample_sheet ${CI_PROJECT_NAME}/data/mouse_splice_realign/sample_sheet.csv" AFTER_NEXTFLOW_CMD: > grep -q 'bam_alignment' ${CI_PROJECT_NAME}/execution/trace.txt && grep -q 'No input BAMs appear to contain splice-aware CIGAR evidence. The workflow will realign all inputs.' .nextflow.log singularity-run: rules: - when: never