Update memory for split_bam and build_minimap_index_transcriptome
This commit is contained in:
Neil Horner
Matt Parker
parent
a33f3b967c
commit
937a41f2cc
@ -4,7 +4,7 @@ include:
|
||||
file: "wf-containers.yaml"
|
||||
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis --ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
||||
--transcriptome_source reference-guided \
|
||||
@ -14,6 +14,7 @@ variables:
|
||||
CI_FLAVOUR: "new"
|
||||
PYTEST_CONTAINER_NAME: "wf-common"
|
||||
PYTEST_CONTAINER_CONFIG_KEY: "common_sha"
|
||||
MEM_CFG: "echo 'process { withName:build_minimap_index { memory = '16.GB' }; withName:build_minimap_index_transcriptome { memory = '16.GB' } }' > ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
|
||||
macos-run:
|
||||
# Let's avoid those ARM64 runners for now
|
||||
@ -41,7 +42,6 @@ docker-run:
|
||||
- ${CI_PROJECT_NAME}/**/*.fna
|
||||
- ${CI_PROJECT_NAME}/**/*.fasta
|
||||
- ${CI_PROJECT_NAME}/**/*.mmi
|
||||
|
||||
|
||||
# Define a 1D job matrix to inject a variable named MATRIX_NAME into
|
||||
# the CI environment, we can use the value of MATRIX_NAME to determine
|
||||
@ -65,38 +65,38 @@ docker-run:
|
||||
when: never
|
||||
- if: $MATRIX_NAME == "isoforms"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "isoforms_bam"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--bam ${CI_PROJECT_NAME}/data/ERR6053095_chr20.bam --transcriptome_source reference-guided \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa --ref_annotation ${CI_PROJECT_NAME}/data/chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,check_annotation_strand,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "no_ref_annotation"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/wf-isoforms_test_data.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/ERR6053095_chr20.fastq --transcriptome_source reference-guided \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/chr20/hg38_chr20.fa \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: run_gffcompare,check_annotation_strand,preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "differential_expression"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome_source reference-guided \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/gencode.v22.annotation.chr20.gtf \
|
||||
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "only_differential_expression"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--transcriptome_source precomputed \
|
||||
--de_analysis \
|
||||
@ -105,13 +105,13 @@ docker-run:
|
||||
--direct_rna --minimap2_index_opts '-k 15' \
|
||||
--ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,faidx,gz_faidx,merge_transcriptomes,merge_gff_bundles,assemble_transcripts,decompress_annotation,decompress_ref,
|
||||
build_minimap_index,get_transcriptome,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "differential_expression_gff3"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--transcriptome_source precomputed \
|
||||
--de_analysis \
|
||||
@ -120,81 +120,81 @@ docker-run:
|
||||
--direct_rna --minimap2_index_opts '-k 15' \
|
||||
--ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "ncbi_gzip"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS:
|
||||
"--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \
|
||||
--direct_rna --minimap2_index_opts '-w 25' \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "ncbi_no_gene_id"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS:
|
||||
"--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
--transcriptome_source precomputed --de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "ensembl_with_versions"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS:
|
||||
"--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/Homo_sapiens.GRCh38.109.gtf.gz \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/Homo_sapiens.GRCh38.cdna.all.fa.gz \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "differential_expression_mouse"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz -C ${CI_PROJECT_NAME}/data/&& wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_mouse/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_mouse.tar.gz -C ${CI_PROJECT_NAME}/data/&& wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS:
|
||||
"--fastq ${CI_PROJECT_NAME}/data/differential_expression_mouse/differential_expression_fastq \
|
||||
--transcriptome_source precomputed --de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_mouse/GRCm39.genome.fa.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_mouse/gencode.vM33.annotation.gtf \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_mouse/gencode.vM33.transcripts.fa.gz \
|
||||
--sample_sheet ${CI_PROJECT_NAME}/data/differential_expression_mouse/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,faidx,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam
|
||||
- if: $MATRIX_NAME == "unstranded_annotation_error"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa --transcriptome_source reference-guided \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression/unstranded_annotation.gtf \
|
||||
--direct_rna --minimap2_index_opts '-k 15' --sample_sheet ${CI_PROJECT_NAME}/data/differential_expression/sample_sheet.csv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: preprocess_reads,faidx,gz_faidx,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
ASSERT_NEXTFLOW_FAILURE: "test_fail" # set to any non-zero length str to allow the nextflow CMD to fail
|
||||
ASSERT_NEXTFLOW_FAILURE_REXP: "In ref_annotation, transcript features must have a strand of either '+' or '-'"
|
||||
- if: $MATRIX_NAME == "igv"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS: "--fastq ${CI_PROJECT_NAME}/data/differential_expression/differential_expression_fastq \
|
||||
--de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression/hg38_chr20.fa \
|
||||
@ -203,22 +203,22 @@ docker-run:
|
||||
--ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression/ref_transcriptome.fasta \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
--igv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,gz_faidx,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome
|
||||
- if: $MATRIX_NAME == "igv_fai_gz"
|
||||
variables:
|
||||
NF_BEFORE_SCRIPT: mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config
|
||||
NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \
|
||||
--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
NF_BEFORE_SCRIPT: "mkdir -p ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/differential_expression_ncbi.tar.gz && tar -xzvf ${CI_PROJECT_NAME}/data/differential_expression_ncbi.tar.gz -C ${CI_PROJECT_NAME}/data/ && wget -O ${CI_PROJECT_NAME}/data/demo.nextflow.config https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config; ${MEM_CFG}"
|
||||
NF_WORKFLOW_OPTS:
|
||||
"--fastq ${CI_PROJECT_NAME}/data/differential_expression_ncbi/differential_expression_fastq \
|
||||
--transcriptome_source precomputed --de_analysis \
|
||||
--ref_genome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \
|
||||
--ref_annotation ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \
|
||||
--direct_rna --ref_transcriptome ${CI_PROJECT_NAME}/data/differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \
|
||||
--sample_sheet test_data/sample_sheet.csv \
|
||||
--igv \
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config"
|
||||
-c ${CI_PROJECT_NAME}/data/demo.nextflow.config -c ${CI_PROJECT_NAME}/data/mm2.config"
|
||||
NF_IGNORE_PROCESSES: >
|
||||
preprocess_reads,merge_transcriptomes,assemble_transcripts,
|
||||
build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome,faidx,gz_faidx
|
||||
|
||||
@ -4,9 +4,13 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
### Changed
|
||||
- `split_bam` and `build_minimap_index_transcriptome` process memory allocation increased.
|
||||
|
||||
## [v1.6.1]
|
||||
### Fixed
|
||||
- merge_gff_compare failing with empty GFF files.
|
||||
- `merge_gff_compare` failing with empty GFF files.
|
||||
|
||||
## [v1.6.0]
|
||||
### Fixed
|
||||
|
||||
3
main.nf
3
main.nf
@ -193,12 +193,11 @@ process split_bam{
|
||||
Partition BAM file into loci or bundles with `params.bundle_min_reads` minimum size
|
||||
If no splitting required, just create single symbolic link to a single bundle.
|
||||
|
||||
Output tuples containing `sample_id` so bundles can be combined later in th pipeline.
|
||||
*/
|
||||
|
||||
label 'isoforms'
|
||||
cpus params.threads
|
||||
memory "4 GB"
|
||||
memory "15 GB"
|
||||
|
||||
input:
|
||||
tuple val(sample_id), path(bam)
|
||||
|
||||
@ -95,7 +95,7 @@ params {
|
||||
]
|
||||
agent = null
|
||||
container_sha = "shad8671ea3a8ed52f2c0f40355e8eb5c6f00d2cbda"
|
||||
common_sha= "shaabceef445fb63214073cbf5836fdd33c04be4ac7"
|
||||
common_sha = "shaabceef445fb63214073cbf5836fdd33c04be4ac7"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -109,7 +109,7 @@ process build_minimap_index_transcriptome{
|
||||
*/
|
||||
label "isoforms"
|
||||
cpus params.threads
|
||||
memory "16 GB"
|
||||
memory "31 GB"
|
||||
input:
|
||||
path reference
|
||||
output:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user