# Include shared CI include: - project: "epi2melabs/ci-templates" file: "wf-containers.yaml" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" CI_FLAVOUR: "new" PYTEST_CONTAINER_NAME: "wf-common" PYTEST_CONTAINER_CONFIG_KEY: "common_sha" PYTEST_TESTS_PATH: "bin/workflow_glue/tests/common" RTEST_CONTAINER_NAME: "wf-transcriptomes-core" RTEST_CONTAINER_CONFIG_KEY: "container_sha" WF_TEMPLATE_ENFORCEMENT_BRANCH: "ruo-next" pytest_wfcontainer: extends: pytest variables: PYTEST_CONTAINER_NAME: "wf-transcriptomes-core" PYTEST_CONTAINER_CONFIG_KEY: "container_sha" PYTEST_TESTS_PATH: "bin/workflow_glue/tests/wf" macos-run: # Let's avoid those ARM64 runners for now tags: - macos - x86 docker-run: tags: - linux - prod - amd64 - eks - xlarge-highio - docker artifacts: when: always paths: - ${CI_PROJECT_NAME} - .nextflow.log exclude: - ${CI_PROJECT_NAME}/**/*.gtf - ${CI_PROJECT_NAME}/**/*.gtf.gz - ${CI_PROJECT_NAME}/**/*.gff3 - ${CI_PROJECT_NAME}/**/*.gff3.gz - ${CI_PROJECT_NAME}/**/*.gff - ${CI_PROJECT_NAME}/**/*.gff.gz - ${CI_PROJECT_NAME}/**/*.fna - ${CI_PROJECT_NAME}/**/*.fasta - ${CI_PROJECT_NAME}/**/*.mmi # Define a 1D job matrix to inject a variable named MATRIX_NAME into # the CI environment, we can use the value of MATRIX_NAME to determine # which options to apply as part of the rules block below # NOTE There is a slightly cleaner way to define this matrix to include # the variables, but it is broken when using long strings! See CW-756 parallel: matrix: - MATRIX_NAME: [ "int_discover_dna", "int_fixed_rna", "int_de_control_vs_control", "smoke_discover", "smoke_fixed", "smoke_direct_rna", "smoke_de", "mouse_de_0countquant" ] rules: # NOTE As we're overriding the rules block for the included docker-run # we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run # being incorrectly scheduled for "detached merge request pipelines" etc. # Guardrail: never schedule docker-run on detached/non-standard branch context. - if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template") when: never # Integration: larger discover-mode run on representative cDNA test bundle. - if: $MATRIX_NAME == "int_discover_dna" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq \ --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf \ --transcriptome_mode discover" NF_IGNORE_PROCESSES: filter_unstranded_annotation,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome # Integration: fixed-annotation + direct-RNA + DE + IGV with GFF input on richer dataset. - if: $MATRIX_NAME == "int_fixed_rna" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \ --sample_sheet test_data/sample_sheet.csv \ --de_analysis \ --direct_rna \ --transcriptome_mode fixed_annotation \ --igv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config " NF_IGNORE_PROCESSES: > gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome # Integration: demo-data control-vs-control to test pathological DE/DTU failure modes and graceful failure handling. # See also unit tests in test_de_analysis.R - if: $MATRIX_NAME == "int_de_control_vs_control" variables: NF_BEFORE_SCRIPT: "\ mkdir -p ${CI_PROJECT_NAME}/data/ \ && echo 'Downlading demo data bundle and config' \ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz \ && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ \ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config \ && echo 'Simulating control-vs-control by copying each sample to a replicate with identical reads' \ && rm -rf ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \ && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode01 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 \ && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode02 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 \ && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode03 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \ && echo 'Finished data preparation step' \ " NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \ --de_analysis \ --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna \ --sample_sheet test_data/sample_sheet.csv \ -c ${CI_PROJECT_NAME}/data/demo.nextflow.config" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json && test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv && test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv && jq -e '.contrasts["condition_treated_vs_control"].dge_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null && jq -e '.contrasts["condition_treated_vs_control"].dtu_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null # Smoke: quick discover-mode sanity check for core cohort outputs. - if: $MATRIX_NAME == "smoke_discover" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf && test -f ${CI_PROJECT_NAME}/samples/sampleA/sampleA.transcriptome.fa # Smoke: fixed-annotation path sanity check for quantification outputs. - if: $MATRIX_NAME == "smoke_fixed" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --transcriptome_mode fixed_annotation" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf && test -f ${CI_PROJECT_NAME}/samples/sampleA/transcript_counts.tsv # Smoke: direct-RNA alignment profile and downstream SQANTI output presence. - if: $MATRIX_NAME == "smoke_direct_rna" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --direct_rna" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/samples/sampleA/alignment/reads.bam && test -f ${CI_PROJECT_NAME}/samples/sampleA/sampleA_sqanti/classification_summary.tsv # Smoke: end-to-end DE/DTU wiring and expected contrast output files. - if: $MATRIX_NAME == "smoke_de" variables: NF_BEFORE_SCRIPT: ":" NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch" AFTER_NEXTFLOW_CMD: > test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv && test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv && [ "$(find ${CI_PROJECT_NAME}/samples -type f -name 'gene_counts.tsv' | wc -l)" -eq 4 ] # Tests a common error when there are 0 annotation counts for a chunk - if: $MATRIX_NAME == "mouse_de_0countquant" variables: NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_subset_test.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz -C ${CI_PROJECT_NAME}/data/" NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/mouse_subset_test/samples \ --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.fa \ --ref_annotation ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.gtf.gz \ --direct_rna --sample_sheet ${CI_PROJECT_NAME}/data/mouse_subset_test/sample_sheet.csv" singularity-run: rules: - when: never