From 5de212cf576e05f1072b844666f8ac12bd80e411 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Wed, 16 Nov 2022 18:49:53 +0000 Subject: [PATCH] Move data to AWS --- .gitlab-ci.yml | 4 ++-- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 2 +- README.md | 45 +++++++++++++++++++++++++++++++---------- docs/quickstart.md | 45 +++++++++++++++++++++++++++++++---------- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ad7b76..fdcb33c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ docker-run: NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes - if: $MATRIX_NAME == "differential_expression" variables: - NF_BEFORE_SCRIPT: tar -xzvf test_data/differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ --de_analysis \ --ref_genome differential_expression/hg38_chr20.fa \ @@ -52,7 +52,7 @@ docker-run: NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes - if: $MATRIX_NAME == "only_differential_expression" variables: - NF_BEFORE_SCRIPT: tar -xzvf test_data/differential_expression.tar.gz + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ --de_analysis \ --ref_genome differential_expression/hg38_chr20.fa \ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 929abcb..1a330bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: pass_filenames: false additional_dependencies: - epi2melabs - - repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/pycqa/flake8 rev: 3.7.9 hooks: - id: flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a941c..415bd91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Updated - Removed sanitize option +- Reduce size of differential expression data. ### Added -- Demo differential expression data in repository. - Improved DE explanation in docs - Option to turn off transcript assembly steps with param transcript_assembly ### Fixed diff --git a/README.md b/README.md index c50354e..6ad9709 100644 --- a/README.md +++ b/README.md @@ -120,15 +120,25 @@ tar -xzvf test_data.tar.gz **Example execution of a workflow for reference-based transcript assembly and fusion detection** ``` OUTPUT=~/output; -nexflow run epi2me-labs/wf-transcriptomes --fastq ERR6053095_chr20.fastq --ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ - --jaffal_refBase chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22" --out_dir outdir -w workspace_dir -profile conda -resume +nexflow run epi2me-labs/wf-transcriptomes \ + --fastq ERR6053095_chr20.fastq \ + --ref_genome chr20/hg38_chr20.fa \ + --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ + --jaffal_refBase chr20/ \ + --jaffal_genome hg38_chr20 \ + --jaffal_annotation "genCode22" \ + --out_dir outdir -w workspace_dir ``` **Example workflow for denovo transcript assembly** ``` OUTPUT=~/output -nextflow run . --fastq test_data/fastq --denovo --ref_genome test_data/SIRV_150601a.fasta -profile local --out_dir ${OUTPUT} -w ${OUTPUT}/workspace \ ---sample sample_id -resume +nextflow run . --fastq test_data/fastq \ + --denovo \ + --ref_genome test_data/SIRV_150601a.fasta \ + --out_dir ${OUTPUT} \ + -w ${OUTPUT}/workspace \ + --sample sample_id ``` A full list of options can be seen in nextflow_schema.json. Below are some commonly used ones. @@ -223,17 +233,30 @@ barcode06,treated ``` You will also need to provide a reference genome and a reference annotation file. -Here is an example cmd to run the workflow using the test_data provided. - +Here is an example cmd to run the workflow. First you will need to download the data with wget. +eg. ``` +wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz OUTPUT=~/output; -nexflow run epi2me-labs/wf-transcriptomes --fastq test_data/differential_expression_fastq \ +nextflow run epi2me-labs/wf-transcriptomes \ + --fastq differential_expression/differential_expression_fastq \ --de_analysis \ - --ref_genome test_data/hg38_chr20.fa \ - --ref_annotation test_data/gencode.v22.annotation.chr20.gtf \ - --direct_rna + --ref_genome differential_expression/hg38_chr20.fa \ + --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ + --direct_rna --minimap_index_opts \-k15 +``` +You can also run the differential expression section of the workflow on its own by providing a reference transcriptome and setting the transcriptome assembly parameter to false. +eg. +``` +nextflow run epi2me-labs/wf-transcriptomes \ + --fastq differential_expression/differential_expression_fastq \ + --de_analysis \ + --ref_genome differential_expression/hg38_chr20.fa \ + --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ + --direct_rna --minimap_index_opts \-k15 \ + --ref_transcriptome differential_expression/ref_transcriptome.fasta \ + --transcriptome_assembly false ``` - ## Workflow outputs * an HTML report document detailing the primary findings of the workflow. diff --git a/docs/quickstart.md b/docs/quickstart.md index 1cda486..c443bdf 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -38,15 +38,25 @@ tar -xzvf test_data.tar.gz **Example execution of a workflow for reference-based transcript assembly and fusion detection** ``` OUTPUT=~/output; -nexflow run epi2me-labs/wf-transcriptomes --fastq ERR6053095_chr20.fastq --ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ - --jaffal_refBase chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22" --out_dir outdir -w workspace_dir -profile conda -resume +nexflow run epi2me-labs/wf-transcriptomes \ + --fastq ERR6053095_chr20.fastq \ + --ref_genome chr20/hg38_chr20.fa \ + --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ + --jaffal_refBase chr20/ \ + --jaffal_genome hg38_chr20 \ + --jaffal_annotation "genCode22" \ + --out_dir outdir -w workspace_dir ``` **Example workflow for denovo transcript assembly** ``` OUTPUT=~/output -nextflow run . --fastq test_data/fastq --denovo --ref_genome test_data/SIRV_150601a.fasta -profile local --out_dir ${OUTPUT} -w ${OUTPUT}/workspace \ ---sample sample_id -resume +nextflow run . --fastq test_data/fastq \ + --denovo \ + --ref_genome test_data/SIRV_150601a.fasta \ + --out_dir ${OUTPUT} \ + -w ${OUTPUT}/workspace \ + --sample sample_id ``` A full list of options can be seen in nextflow_schema.json. Below are some commonly used ones. @@ -141,17 +151,30 @@ barcode06,treated ``` You will also need to provide a reference genome and a reference annotation file. -Here is an example cmd to run the workflow using the test_data provided. - +Here is an example cmd to run the workflow. First you will need to download the data with wget. +eg. ``` +wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz OUTPUT=~/output; -nexflow run epi2me-labs/wf-transcriptomes --fastq test_data/differential_expression_fastq \ +nextflow run epi2me-labs/wf-transcriptomes \ + --fastq differential_expression/differential_expression_fastq \ --de_analysis \ - --ref_genome test_data/hg38_chr20.fa \ - --ref_annotation test_data/gencode.v22.annotation.chr20.gtf \ - --direct_rna + --ref_genome differential_expression/hg38_chr20.fa \ + --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ + --direct_rna --minimap_index_opts \-k15 +``` +You can also run the differential expression section of the workflow on its own by providing a reference transcriptome and setting the transcriptome assembly parameter to false. +eg. +``` +nextflow run epi2me-labs/wf-transcriptomes \ + --fastq differential_expression/differential_expression_fastq \ + --de_analysis \ + --ref_genome differential_expression/hg38_chr20.fa \ + --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ + --direct_rna --minimap_index_opts \-k15 \ + --ref_transcriptome differential_expression/ref_transcriptome.fasta \ + --transcriptome_assembly false ``` - ## Workflow outputs * an HTML report document detailing the primary findings of the workflow.