wf-transcriptomes-v202/.gitlab-ci.yml

# Include shared CI
include:
    - project: "epi2melabs/ci-templates"
      file: "wf-containers.yaml"

variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/  &&  wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config"
              NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
              --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
              --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
              --direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
              -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
              CI_FLAVOUR: "new"
              PYTEST_CONTAINER_NAME: "wf-common"
              PYTEST_CONTAINER_CONFIG_KEY: "common_sha"
              PYTEST_TESTS_PATH: "bin/workflow_glue/tests/common"
              RTEST_CONTAINER_NAME: "wf-transcriptomes-core"
              RTEST_CONTAINER_CONFIG_KEY: "container_sha"
              WF_TEMPLATE_ENFORCEMENT_BRANCH: "ruo-next"

pytest_wfcontainer:
    extends: pytest
    variables:
              PYTEST_CONTAINER_NAME: "wf-transcriptomes-core"
              PYTEST_CONTAINER_CONFIG_KEY: "container_sha"
              PYTEST_TESTS_PATH: "bin/workflow_glue/tests/wf"

macos-run:
  # Let's avoid those ARM64 runners for now
  tags:
    - macos
    - x86


docker-run:
    tags:
        - linux
        - prod
        - amd64
        - eks
        - xlarge-highio
        - docker
    artifacts:
        when: always
        paths:
            - ${CI_PROJECT_NAME}
            - .nextflow.log
        exclude:
            - ${CI_PROJECT_NAME}/**/*.gtf
            - ${CI_PROJECT_NAME}/**/*.gtf.gz
            - ${CI_PROJECT_NAME}/**/*.gff3
            - ${CI_PROJECT_NAME}/**/*.gff3.gz
            - ${CI_PROJECT_NAME}/**/*.gff
            - ${CI_PROJECT_NAME}/**/*.gff.gz
            - ${CI_PROJECT_NAME}/**/*.fna
            - ${CI_PROJECT_NAME}/**/*.fasta
            - ${CI_PROJECT_NAME}/**/*.mmi


    # Define a 1D job matrix to inject a variable named MATRIX_NAME into
    #   the CI environment, we can use the value of MATRIX_NAME to determine
    #   which options to apply as part of the rules block below
    # NOTE There is a slightly cleaner way to define this matrix to include
    #   the variables, but it is broken when using long strings! See CW-756
    parallel:
        matrix:
            - MATRIX_NAME: [
                "int_discover_dna", "int_fixed_rna", "int_de_control_vs_control",
                "smoke_discover", "smoke_fixed", "smoke_direct_rna", "smoke_de",
                "mouse_de_0countquant"
            ]
    rules:
        # NOTE As we're overriding the rules block for the included docker-run
        #   we must redefine this CI_COMMIT_BRANCH rule to prevent docker-run
        #   being incorrectly scheduled for "detached merge request pipelines" etc.
        # Guardrail: never schedule docker-run on detached/non-standard branch context.
        - if: ($CI_COMMIT_BRANCH == null || $CI_COMMIT_BRANCH == "dev-template")
          when: never

        # Integration: larger discover-mode run on representative cDNA test bundle.
        - if: $MATRIX_NAME == "int_discover_dna"
          variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ &&  wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
              NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq \
                  --ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
                  --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf \
                  --transcriptome_mode discover"
              NF_IGNORE_PROCESSES: filter_unstranded_annotation,validate_ref_annotation,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome

        # Integration: fixed-annotation + direct-RNA + DE + IGV with GFF input on richer dataset.
        - if: $MATRIX_NAME == "int_fixed_rna"
          variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/  &&  wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config;"
              NF_WORKFLOW_OPTS: "--fastq  ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
                  --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
                  --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gff \
                  --sample_sheet test_data/sample_sheet.csv \
                  --de_analysis \
                  --direct_rna \
                  --transcriptome_mode fixed_annotation \
                  --igv \
                  -c ${CI_PROJECT_NAME}/data/demo.nextflow.config "
              NF_IGNORE_PROCESSES: >
                gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
                build_minimap_index,validate_ref_annotation,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome                
        
        # Integration: demo-data control-vs-control to test pathological DE/DTU failure modes and graceful failure handling.
        #              See also unit tests in test_de_analysis.R
        - if: $MATRIX_NAME == "int_de_control_vs_control"
          variables:
              NF_BEFORE_SCRIPT: "\
                  mkdir -p ${CI_PROJECT_NAME}/data/ \
                  && echo 'Downlading demo data bundle and config' \
                  && wget -nv -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz \
                  && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ \
                  && wget -nv -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config \
                  && echo 'Simulating control-vs-control by copying each sample to a replicate with identical reads' \
                  && rm -rf ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \
                  && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode01 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode04 \
                  && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode02 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode05 \
                  && cp -R ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode03 ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq/barcode06 \
                  && echo 'Finished data preparation step' \
                  "
              NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
                  --de_analysis \
                  --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
                  --ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
                  --direct_rna \
                  --sample_sheet test_data/sample_sheet.csv \
                  -c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
              AFTER_NEXTFLOW_CMD: >
                  test -f ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json &&
                  test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv &&
                  test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv &&
                  jq -e '.contrasts["condition_treated_vs_control"].dge_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null &&
                  jq -e '.contrasts["condition_treated_vs_control"].dtu_status | IN("SUCCESS", "FAILED")' ${CI_PROJECT_NAME}/de_analysis/de_qc_stats.json >/dev/null                  

        # Smoke: quick discover-mode sanity check for core cohort outputs.
        - if: $MATRIX_NAME == "smoke_discover"
          variables:
              NF_BEFORE_SCRIPT: ":"
              NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf"
              AFTER_NEXTFLOW_CMD: >
                  test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf &&
                  test -f ${CI_PROJECT_NAME}/samples/sampleA/sampleA.transcriptome.fa                  

        # Smoke: fixed-annotation path sanity check for quantification outputs.
        - if: $MATRIX_NAME == "smoke_fixed"
          variables:
              NF_BEFORE_SCRIPT: ":"
              NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --transcriptome_mode fixed_annotation"
              AFTER_NEXTFLOW_CMD: >
                  test -f ${CI_PROJECT_NAME}/samples/sampleA/transcripts.gtf &&
                  test -f ${CI_PROJECT_NAME}/samples/sampleA/transcript_counts.tsv                  

        # Smoke: direct-RNA alignment profile and downstream SQANTI output presence.
        - if: $MATRIX_NAME == "smoke_direct_rna"
          variables:
              NF_BEFORE_SCRIPT: ":"
              NF_WORKFLOW_OPTS: "--fastq test_data/smoke/reads.fastq --sample sampleA --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --direct_rna"
              AFTER_NEXTFLOW_CMD: >
                  test -f ${CI_PROJECT_NAME}/samples/sampleA/alignment/reads.bam &&
                  test -f ${CI_PROJECT_NAME}/samples/sampleA/sqanti/classification_summary.tsv                  

        # Smoke: end-to-end DE/DTU wiring and expected contrast output files.
        - if: $MATRIX_NAME == "smoke_de"
          variables:
              NF_BEFORE_SCRIPT: ":"
              NF_WORKFLOW_OPTS: "--fastq test_data/smoke/de --sample_sheet test_data/smoke/sample_sheet_de.csv --ref_genome test_data/smoke/reference.fa --ref_annotation test_data/smoke/annotation.gtf --de_analysis --reference_level control --covariates batch"
              AFTER_NEXTFLOW_CMD: >
                  test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dge.tsv &&
                  test -f ${CI_PROJECT_NAME}/de_analysis/condition_treated_vs_control/results_dtu_transcript.tsv &&
                  [ "$(find ${CI_PROJECT_NAME}/samples -type f -name 'gene_counts.tsv' | wc -l)" -eq 4 ]                  
        # Tests a common error when there are 0 annotation counts for a chunk
        - if: $MATRIX_NAME == "mouse_de_0countquant"
          variables:
              NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/mouse_subset_test.tar.gz -O ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/mouse_subset_test.tar.gz  -C ${CI_PROJECT_NAME}/data/" 
              NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/mouse_subset_test/samples \
                  --de_analysis --ref_genome ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.fa \
                  --ref_annotation ${CI_PROJECT_NAME}/data/mouse_subset_test/mouse_subset.gtf.gz \
                --direct_rna --sample_sheet ${CI_PROJECT_NAME}/data/mouse_subset_test/sample_sheet.csv"


singularity-run:
    rules: 
        - when: never