146 lines
11 KiB
YAML
146 lines
11 KiB
YAML
# Include shared CI
|
|
include:
|
|
- project: "epi2melabs/ci-templates"
|
|
file: "wf-containers.yaml"
|
|
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
|
--de_analysis --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
|
|
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
|
|
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
|
CI_FLAVOUR: "new"
|
|
PYTEST_CONTAINER_NAME: "wf-common"
|
|
PYTEST_CONTAINER_CONFIG_KEY: "common_sha"
|
|
RTEST_CONTAINER_NAME: "wf-transcriptomes-core"
|
|
RTEST_CONTAINER_CONFIG_KEY: "container_sha"
|
|
WF_TEMPLATE_ENFORCEMENT_BRANCH: "CW-6552"
|
|
|
|
macos-run:
|
|
# Let's avoid those ARM64 runners for now
|
|
tags:
|
|
- macos
|
|
- x86
|
|
|
|
|
|
docker-run:
|
|
tags:
|
|
- linux
|
|
- prod
|
|
- amd64
|
|
- eks
|
|
- xlarge-highio
|
|
- docker
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- ${CI_PROJECT_NAME}
|
|
- .nextflow.log
|
|
exclude:
|
|
- ${CI_PROJECT_NAME}/**/*.gtf
|
|
- ${CI_PROJECT_NAME}/**/*.gtf.gz
|
|
- ${CI_PROJECT_NAME}/**/*.gff3
|
|
- ${CI_PROJECT_NAME}/**/*.gff3.gz
|
|
- ${CI_PROJECT_NAME}/**/*.gff
|
|
- ${CI_PROJECT_NAME}/**/*.gff.gz
|
|
- ${CI_PROJECT_NAME}/**/*.fna
|
|
- ${CI_PROJECT_NAME}/**/*.fasta
|
|
- ${CI_PROJECT_NAME}/**/*.mmi
|
|
|
|
|
|
# Define a 1D job matrix to inject a variable named MATRIX_NAME into
|
|
# the CI environment, we can use the value of MATRIX_NAME to determine
|
|
# which options to apply as part of the rules block below
|
|
# NOTE There is a slightly cleaner way to define this matrix to include
|
|
# the variables, but it is broken when using long strings! See CW-756
|
|
parallel:
|
|
matrix:
|
|
- MATRIX_NAME: [
|
|
"de-poscounts-fallback", "discover", "igv",
|
|
"smoke_discover", "smoke_fixed", "smoke_direct_rna", "smoke_de",
|
|
"no_annotation", "invalid_mode", "conflicting_flags"
|
|
]
|
|
rules:
|
|
# NOTE As we're overriding the rules block for the included docker-run
|
|
# we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run
|
|
# being incorrectly scheduled for "detached merge request pipelines" etc.
|
|
- if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template")
|
|
when: never
|
|
- if: $MATRIX_NAME == "discover"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf"
|
|
NF_IGNORE_PROCESSES: filter_unstranded_annotation,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
|
- if: $MATRIX_NAME == "no_annotation"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
|
|
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config "
|
|
ASSERT_NEXTFLOW_FAILURE: "1"
|
|
ASSERT_NEXTFLOW_FAILURE_REXP: "Missing required parameter: --ref_annotation"
|
|
- if: $MATRIX_NAME == "de-poscounts-fallback"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
|
--de_analysis \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
|
|
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
|
|
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config "
|
|
NF_IGNORE_PROCESSES: faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
|
AFTER_NEXTFLOW_CMD: >
|
|
grep -Eq '"deseq2_size_factor_method": "poscounts"' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json
|
|
- if: $MATRIX_NAME == "only_differential_expression"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
|
|
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
|
--de_analysis \
|
|
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
|
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \
|
|
--direct_rna --transcriptome_mode fixed_annotation --minimap2_index_opts '-k 15' \
|
|
--sample_sheet test_data/sample_sheet.csv \
|
|
--igv \
|
|
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config "
|
|
NF_IGNORE_PROCESSES: >
|
|
gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
|
|
build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
|
- if: $MATRIX_NAME == "smoke_discover"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/cohort/transcripts.gtf &&
|
|
test -f ${CI_PROJECT_NAME}/cohort/cohort.transcriptome.fa &&
|
|
test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf
|
|
- if: $MATRIX_NAME == "smoke_fixed"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --transcriptome_mode fixed_annotation"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/cohort/transcripts.gtf &&
|
|
test -f ${CI_PROJECT_NAME}/cohort/transcript_counts.tsv
|
|
- if: $MATRIX_NAME == "smoke_direct_rna"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --direct_rna"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/cohort/alignments/sampleA/reads.bam &&
|
|
test -f ${CI_PROJECT_NAME}/cohort/sqanti_cohort/classification_summary.tsv
|
|
- if: $MATRIX_NAME == "smoke_de"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch"
|
|
AFTER_NEXTFLOW_CMD: >
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv &&
|
|
test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv &&
|
|
[ "$(find ${CI_PROJECT_NAME}/samples -type f -name 'gene_counts.tsv' | wc -l)" -eq 4 ]
|
|
- if: $MATRIX_NAME == "invalid_mode"
|
|
variables:
|
|
NF_BEFORE_SCRIPT: ":"
|
|
NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --transcriptome_mode nonsense"
|
|
ASSERT_NEXTFLOW_FAILURE: "1"
|
|
ASSERT_NEXTFLOW_FAILURE_REXP: "nonsense is not a valid choice"
|