From 4bb02049ecf2fdce06b4fba3212cf9a545e51f33 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Mon, 12 Feb 2024 10:20:02 +0000 Subject: [PATCH] Resource updates --- .gitlab-ci.yml | 55 ++++++++++++++++++------------ CHANGELOG.md | 1 + README.md | 2 +- docs/03_compute_requirements.md | 2 +- main.nf | 18 +++++----- nextflow_schema.json | 2 +- subworkflows/reference_assembly.nf | 2 +- 7 files changed, 47 insertions(+), 35 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eef3466..f358225 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,13 +4,14 @@ include: file: "wf-containers.yaml" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ --de_analysis --ref_genome differential_expression/hg38_chr20.fa \ --transcriptome-source reference-guided \ --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ --direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \ - --jaffal_refBase differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22" + --jaffal_refBase differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \ + -c demo.nextflow.config" CI_FLAVOUR: "new" macos-run: @@ -62,35 +63,39 @@ docker-run: when: never - if: $MATRIX_NAME == "isoforms" variables: - NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz + NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ERR6053095_chr20.fastq --transcriptome-source reference-guided \ - --ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm" + --ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf --pychopper_backend phmm \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "no_ref_annotation" variables: - NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz + NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ERR6053095_chr20.fastq --transcriptome-source reference-guided \ - --ref_genome chr20/hg38_chr20.fa" + --ref_genome chr20/hg38_chr20.fa \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: run_gffcompare,preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "fusions" variables: - NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz + NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq ERR6053095_chr20.fastq --transcriptome-source reference-guided \ --ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ - --jaffal_refBase chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22" + --jaffal_refBase chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ --de_analysis \ --ref_genome differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \ --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ - --direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv" + --direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "only_differential_expression" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ --transcriptome-source precomputed \ --de_analysis \ @@ -98,13 +103,14 @@ docker-run: --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff \ --direct_rna --minimap2_index_opts '-k 15' \ --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --sample_sheet test_data/sample_sheet.csv" + --sample_sheet test_data/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_gff3" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ --transcriptome-source precomputed \ --de_analysis \ @@ -112,13 +118,14 @@ docker-run: --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff3 \ --direct_rna --minimap2_index_opts '-k 15' \ --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --sample_sheet test_data/sample_sheet.csv" + --sample_sheet test_data/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ncbi_gzip" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf differential_expression_ncbi.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf differential_expression_ncbi.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq differential_expression_ncbi/differential_expression_fastq \ --transcriptome-source precomputed \ @@ -126,46 +133,50 @@ docker-run: --ref_genome differential_expression_ncbi/GRCh38.p14.NCBI_test.fna.gz \ --ref_annotation differential_expression_ncbi/GRCh38.p14_NCBI_test.gtf.gz \ --direct_rna --minimap2_index_opts '-w 25' \ - --sample_sheet test_data/sample_sheet.csv" + --sample_sheet test_data/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ncbi_no_gene_id" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf differential_expression_ncbi.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression_ncbi.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_ncbi.tar.gz && tar -xzvf differential_expression_ncbi.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq differential_expression_ncbi/differential_expression_fastq \ --transcriptome-source precomputed --de_analysis \ --ref_genome differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.fna.gz \ --ref_annotation differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_genomic.gff.gz \ --direct_rna --ref_transcriptome differential_expression_ncbi/GCF_000001405.40_GRCh38.p14_rna.fna.gz \ - --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv" + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "ensembl_with_versions" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq differential_expression/differential_expression_fastq \ --transcriptome-source precomputed --de_analysis \ --ref_genome differential_expression/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz \ --ref_annotation differential_expression/Homo_sapiens.GRCh38.109.gtf.gz \ --direct_rna --ref_transcriptome differential_expression/Homo_sapiens.GRCh38.cdna.all.fa.gz \ - --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv" + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "differential_expression_mouse" variables: - NF_BEFORE_SCRIPT: wget -O differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf differential_expression_mouse.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression_mouse.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression_mouse.tar.gz && tar -xzvf differential_expression_mouse.tar.gz && wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/demo.nextflow.config NF_WORKFLOW_OPTS: "-executor.\\$$local.memory 16GB \ --fastq differential_expression_mouse/differential_expression_fastq \ --transcriptome-source precomputed --de_analysis \ --ref_genome differential_expression_mouse/GRCm39.genome.fa.gz \ --ref_annotation differential_expression_mouse/gencode.vM33.annotation.gtf \ --direct_rna --ref_transcriptome differential_expression_mouse/gencode.vM33.transcripts.fa.gz \ - --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv" + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv \ + -c demo.nextflow.config" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam diff --git a/CHANGELOG.md b/CHANGELOG.md index 660d425..48be511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Renamed files: - `de_analysis/all_counts_filtered.tsv` to `de_analysis/filtered_transcript_counts_with_genes.tsv` - `de_analysis/de_tpm_transcript_counts.tsv` to `de_analysis/unfiltered_tpm_transcript_counts.tsv` +- Minimum memory requirements to `32 GB`. ### Added - Published isoforms table to output directory. - Output additional `de_analysis/cpm_gene_counts.tsv` with counts per million gene counts. diff --git a/README.md b/README.md index ed4fe8b..8b1b42e 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Recommended requirements: Minimum requirements: + CPUs = 8 -+ Memory = 16GB ++ Memory = 32GB Approximate run time: 15 minutes per sample, with 1 million reads and recommended resources. diff --git a/docs/03_compute_requirements.md b/docs/03_compute_requirements.md index ff522ea..1371fb9 100644 --- a/docs/03_compute_requirements.md +++ b/docs/03_compute_requirements.md @@ -6,7 +6,7 @@ Recommended requirements: Minimum requirements: + CPUs = 8 -+ Memory = 16GB ++ Memory = 32GB Approximate run time: 15 minutes per sample, with 1 million reads and recommended resources. diff --git a/main.nf b/main.nf index e23745b..5282271 100644 --- a/main.nf +++ b/main.nf @@ -19,7 +19,7 @@ OPTIONAL_FILE = file("$projectDir/data/OPTIONAL_FILE") process getVersions { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" output: path "versions.txt" script: @@ -44,7 +44,7 @@ process getVersions { process getParams { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" output: path "params.json" script: @@ -60,7 +60,7 @@ process getParams { process decompress_ref { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" input: path compressed_ref output: @@ -74,7 +74,7 @@ process decompress_ref { process decompress_annotation { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" input: path compressed_annotation output: @@ -89,7 +89,7 @@ process decompress_annotation { process decompress_transcriptome { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" input: path "compressed_ref.gz" output: @@ -104,7 +104,7 @@ process decompress_transcriptome { process preprocess_ref_annotation { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" input: path ref_annotation output: @@ -119,7 +119,7 @@ process preprocess_ref_annotation { process preprocess_ref_transcriptome { label "isoforms" cpus 1 - memory "500MB" + memory "2 GB" input: path "ref_transcriptome" output: @@ -177,7 +177,7 @@ process build_minimap_index{ */ label "isoforms" cpus params.threads - memory "16 GB" + memory "31 GB" input: path reference @@ -367,7 +367,7 @@ process merge_transcriptomes { // Merge the transcriptomes from all samples label 'isoforms' cpus 2 - memory "4 GB" + memory "2 GB" input: path "query_annotations/*" path ref_annotation diff --git a/nextflow_schema.json b/nextflow_schema.json index 69481fd..155299a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -388,7 +388,7 @@ }, "minimum": { "cpus": 8, - "memory": "16GB" + "memory": "32GB" }, "run_time": "15 minutes per sample, with 1 million reads and recommended resources.", "arm_support": false diff --git a/subworkflows/reference_assembly.nf b/subworkflows/reference_assembly.nf index 7ab994a..aa8f8f5 100644 --- a/subworkflows/reference_assembly.nf +++ b/subworkflows/reference_assembly.nf @@ -6,7 +6,7 @@ process map_reads{ */ label "isoforms" cpus params.threads - memory "16 GB" + memory "31 GB" input: tuple val(sample_id), path (fastq_reads), path(index), path(reference)