From 045a077fe1e25e0f831c61d3bf441440d9ed7df8 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Fri, 1 Dec 2023 12:36:33 +0000 Subject: [PATCH] docs update --- .gitlab-ci.yml | 18 +- .pre-commit-config.yaml | 12 +- README.md | 434 +++++++++++++----------- docs/01_brief_description.md | 1 + docs/02_introduction.md | 8 + docs/03_compute_requirements.md | 13 + docs/04_install_and_run.md | 34 ++ docs/05_related_protocols.md | 3 + docs/06_inputs.md | 70 ++++ docs/07_outputs.md | 28 ++ docs/08_pipeline_overview.md | 76 +++++ docs/09_troubleshooting.md | 2 + docs/10_FAQ.md | 3 + docs/11_other.md | 3 + docs/header.md | 4 - docs/intro.md | 64 ---- docs/links.md | 6 - docs/quickstart.md | 194 ----------- main.nf | 28 +- nextflow.config | 12 +- nextflow_schema.json | 30 +- output_definition.json | 197 +++++++++++ subworkflows/JAFFAL/gene_fusions.nf | 2 + subworkflows/differential_expression.nf | 13 + subworkflows/reference_assembly.nf | 1 + 25 files changed, 748 insertions(+), 508 deletions(-) create mode 100644 docs/01_brief_description.md create mode 100644 docs/02_introduction.md create mode 100644 docs/03_compute_requirements.md create mode 100644 docs/04_install_and_run.md create mode 100644 docs/05_related_protocols.md create mode 100644 docs/06_inputs.md create mode 100644 docs/07_outputs.md create mode 100644 docs/08_pipeline_overview.md create mode 100644 docs/09_troubleshooting.md create mode 100644 docs/10_FAQ.md create mode 100644 docs/11_other.md delete mode 100644 docs/header.md delete mode 100644 docs/intro.md delete mode 100644 docs/links.md delete mode 100644 docs/quickstart.md create mode 100644 output_definition.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5a391b..56d4579 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,11 +4,13 @@ include: file: "wf-containers.yaml" variables: - NF_BEFORE_SCRIPT: wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz && tar -xzvf test_data.tar.gz - NF_WORKFLOW_OPTS: "--fastq ERR6053095_chr20.fastq \ - --ref_genome chr20/hg38_chr20.fa --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ - --jaffal_refBase chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22" - NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes + NF_BEFORE_SCRIPT: wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz + NF_WORKFLOW_OPTS: "--fastq differential_expression/differential_expression_fastq \ + --de_analysis --ref_genome differential_expression/hg38_chr20.fa \ + --transcriptome-source reference-guided \ + --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ + --direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \ + --jaffal_refBase differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22" CI_FLAVOUR: "new" macos-run: @@ -17,6 +19,10 @@ macos-run: - macos - x86 +aws-run: + artifacts: + when: always + docker-run: artifacts: when: always @@ -74,7 +80,7 @@ docker-run: --de_analysis \ --ref_genome differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \ --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ - --direct_rna --minimap2_index_opts '-k 15' --sample_sheet test_data/sample_sheet.csv" + --direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv" NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref,decompress_transcriptome,preprocess_ref_transcriptome - if: $MATRIX_NAME == "only_differential_expression" variables: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6583953..5cda947 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,14 @@ repos: - repo: local hooks: - - id: docs_schema - name: docs_schema - entry: parse_docs -p docs -e .md -s intro links -oj nextflow_schema.json - language: python - always_run: true - pass_filenames: false - additional_dependencies: - - epi2melabs - id: docs_readme name: docs_readme - entry: parse_docs -p docs -e .md -s header intro quickstart links -ot README.md + entry: parse_docs -p docs -e .md -s 01_brief_description 02_introduction 03_compute_requirements 04_install_and_run 05_related_protocols 06_inputs 07_outputs 08_pipeline_overview 09_troubleshooting 10_FAQ 11_other -ot README.md -od output_definition.json -ns nextflow_schema.json language: python always_run: true pass_filenames: false additional_dependencies: - - epi2melabs + - epi2melabs>=0.0.48 - id: build_models name: build_models entry: datamodel-codegen --strict-nullable --base-class workflow_glue.results_schema_helpers.BaseModel --use-schema-description --disable-timestamp --input results_schema.yml --input-file-type openapi --output bin/workflow_glue/results_schema.py diff --git a/README.md b/README.md index 7d86e80..f7c4a04 100644 --- a/README.md +++ b/README.md @@ -1,199 +1,242 @@ -# wf-transcriptomes +# Workflow Transcriptomes -This repository contains a [nextflow](https://www.nextflow.io/) workflow -for assembly and annotation of transcripts from Oxford Nanopore cDNA or direct RNA reads. +Transcriptome analysis including assembly and annotation of cDNA and direct RNA sequencing data, gene fusions and differential expression. ## Introduction -This workflow identifies RNA isoforms using either cDNA or direct RNA (dRNA) -Oxford Nanopore reads. +This workflow can be used for the following: -### Preprocesing -cDNA reads are initially preprocessed by [pychopper](https://github.com/epi2me-labs/pychopper) -for the identification of full-length reads, as well as trimming and orientation correction (This step is omitted for - direct RNA reads). ++ Identify RNA transcripts using either cDNA or direct RNA reads. ++ Reference aided transcriptome assembly. ++ Annotation of assembled transcripts. ++ Gene fusions detection. ++ Differential gene expression analysis using a pre-computed or assembled reference transcriptome. ++ Differential transcript usage analysis using a precomputed or assembled reference transcriptome. -### Transcript assembly -#### Reference-aided transcript assembly approach -* Full length reads are mapped to a supplied reference genome using [minimap2](https://github.com/lh3/minimap2) -* Transcripts are assembled by [stringtie](http://ccb.jhu.edu/software/stringtie) -in long read mode (with or without a guide reference annotation) to generate the GFF annotation. -* The annotation generated by the pipeline is compared to the reference annotation. -using [gffcompare](http://ccb.jhu.edu/software/stringtie/gffcompare.shtml) +## Compute requirements -### Fusion gene detection -Fusion gene detection is performed using [JAFFA](https://github.com/Oshlack/JAFFA), with the JAFFAL extension for use -with ONT long reads. +Recommended requirements: -### Differential expression analysis ++ CPUs = 16 ++ memory = 32GB -Differential gene expression (DGE) and differential transcript usage (DTU) analyses aim to identify genes and/or transcripts that show statistically altered expression patterns in a studied biological system. The results of the differential analyses are presented in a quantitative format and therefore the degree of change (up or down regulation) between experimental conditions can be calculated for each gene identified. +Minimum requirement: -These differential analyses work by taking a “snapshot” of mRNA abundance and calculating the relative levels of transcripts and isoforms. In this context, expression corresponds to the number of messenger RNAs (mRNA) measured from each gene isoform within the organism / tissue / culture being investigated. In order to determine expression levels across the whole genome, sequence data specifically targeting the mRNA molecules can be generated. ++ CPUs = 8 ++ memory = 16GB -Oxford Nanopore Technologies provides a number of sequencing solutions to allow users to generate the required snapshot of gene expression. This can be achieved by both sequencing the mRNA directly, or via a complementary DNA (cDNA) proxy. In contrast to short read sequencing technologies, entire mRNA transcripts can be captured as single reads. The example data provided with this tutorial is from a study based on the PCR-cDNA kit. This is a robust choice for performing differential transcript usage studies. This kit is suitable for preparation of sequence libraries from low mRNA input quantities. The cDNA population is enriched through PCR with low bias; an important prerequisite for the subsequent statistical analysis. +Approximate run time: 15 minutes per sample, with 1 million reads and recommended resources. -[Workflow-transcriptomes](https://github.com/epi2me-labs/wf-transcriptomes) includes a subworkflow for DGE and DTU. The first step involves using either a reference alignment or _de novo_ assembly approach to create a set of mRNA sequences per sample. These are merged into a non-redundant transcriptome using [stringtie merge](http://ccb.jhu.edu/software/stringtie). The reads are then aligned to the transcriptome using minimap2 in a splice-aware manner. [Salmon](https://github.com/COMBINE-lab/salmon) is used for transcript quantification, giving per transcript counts and then the following R packages are used for analysis. - -### Pre-filtering of quantitative data using DRIMSeq -DRIMSeq (Nowicka and Robinson (2016)) is used to filter the transcript count data from the salmon analysis. The filter step will be used to select for genes and transcripts that satisfy rules for the number of samples in which a gene or transcript must be observed and minimum threshold levels for the number of observed reads. The parameters used for filtering are defined in the config.yaml file. The default parameters defined for this analysis include -* min_samps_gene_expr = 3 - a transcript must be mapped to a gene in at least this minimum number of samples for the gene be included in the analysis -* min_samps_feature_expr = 1 - a transcript must be mapped to an isoform in at least this this minimum number of samples for the gene isoform to be included in the analysis -* min_gene_expr = 10 - the minimum number of total mapped sequence reads for a gene to be considered expressed -* min_feature_expr = 3 - the minimum number of total mapped sequence reads for a gene isoform to be considered - -### edgeR based differential expression analysis -+A statistical analysis is first performed using edgeR (Robinson, McCarthy, and Smyth (2010), McCarthy et al. (2012)) to identify the subset of differentially expressed genes. The filtered list of gene counts is used as input. A normalisation factor is calculated for each sequence library (using the default TMM method - please see McCarthy et al. (2012) for further details). The defined experimental design is used to calculate estimates of dispersion for each of the gene features. Statistical tests are calculated using the contrasts defined in the experimental design. The differentially expressed genes are corrected for false discovery (fdr) using the method of Benjamini & Hochberg (Benjamini and Hochberg (1995)) - -### Differential transcript usage using DEXSeq -Differential transcript usage analysis is performed using the R DEXSeq package (Reyes et al. (2013)). Similar to the edgeR package, DEXSeq estimates the variance between the biological replicates and applies generalised linear models for the statistical testing. The key difference is that the DEXSeq method looks for differences at the exon count level. DEXSeq uses the filtered transcript count data prepared earlier in this analysis. - -### StageR stage-wise analysis of DGE and DTU -The final component of this isoform analysis is a stage-wise statistical test using the R software package `stageR` (Van den Berge and Clement (2018)). stageR uses (1) the raw p-values for DTU from the DEXSeq analysis in the previous section and (2) a false-discovery corrected set of p-values from testing whether individual genes contain at least one exon showing DTU. A hierarchical two-stage statistical testing evaluates the set of genes for DTU. - -## Running the workflow -For the differential expression analysis section you should have at least 3 repeats for each sample. -Your FASTQ data will need to be organised in to 6 directories that represent 3 repeats for each condition. - - -## Analysis -Differential gene expression is sensitive to the input data quantity and quality. There should be equivalence between samples in the number of sequence reads, mapped reads and quality scores. The sequence and alignment summary plots in the report can be used to assess these metrics. There is also a table that shows the transcript per million(TPM) calculated from the salmon counts. TPM normalizes the data for gene length and then sequencing depth, and makes it easier to compare across samples compared to counts. - -### Workflow inputs -- Directory containing cDNA/direct RNA reads. Or a directory containing subdirectories each with reads from different samples - (in fastq/fastq.gz format) -- Reference genome in fasta format (required for reference-based assembly). -- Optional reference annotation in GFF2/3 format (extensions allowed are .gtf(.gz), .gff(.gz), .gff3(.gz)) (required for differential expression analysis `--de_analysis`). Only annotation files from [Encode](https://www.encodeproject.org), [Ensembl](https://www.ensembl.org/index.html) and [NCBI](https://www.ncbi.nlm.nih.gov/) are supported. -- For fusion detection, JAFFAL reference files (see Quickstart) +ARM processor support: False -## Quickstart +## Install and run -The workflow uses [nextflow](https://www.nextflow.io/) to manage compute and -software resources, as such nextflow will need to be installed before attempting -to run the workflow. +These are instructions to install and run the workflow on command line. You can also access the workflow via the [EPI2ME application](https://labs.epi2me.io/downloads/). -The workflow can currently be run using either -[Docker](https://www.docker.com/products/docker-desktop), -[Singularity](https://sylabs.io/singularity/) to provide isolation of -the required software. Each method is automated out-of-the-box provided -either docker or singularity is installed. +The workflow uses [Nextflow](https://www.nextflow.io/) to manage compute and software resources, therefore nextflow will need to be installed before attempting to run the workflow. + +The workflow can currently be run using either [Docker](https://www.docker.com/products/docker-desktop) or +[Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html) to provide isolation of +the required software. Both methods are automated out-of-the-box provided +either docker or singularity is installed. This is controlled by the [`-profile`](https://www.nextflow.io/docs/latest/config.html#config-profiles) parameter as exemplified below. It is not required to clone or download the git repository in order to run the workflow. -For more information on running EPI2ME Labs workflows [visit out website](https://labs.epi2me.io/wfindex). +More information on running EPI2ME workflows can be found on our [website](https://labs.epi2me.io/wfindex). - -### Workflow options - -To obtain the workflow, having installed `nextflow`, users can run: +The following command can be used to obtain the workflow. This will pull the repository in to the assets folder of nextflow and provide a list of all parameters available for the workflow as well as an example command: ``` -nextflow run epi2me-labs/wf-transcriptomes --help +nextflow run epi2me-labs/wf-transcriptomes -–help ``` - -to see the options for the workflow. - -**Download demonstration data** - -A small test dataset is provided for the purposes of testing the workflow software. It consists of reads, reference, -and annotations from human chromosome 20 only. -It can be downloaded using: -```shell -wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz -tar -xzvf test_data.tar.gz +A demo dataset is provided for testing of the workflow. It can be downloaded using: ``` - -**Example execution of a workflow for reference-based transcript assembly and fusion detection** +wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz +tar -xzvf differential_expression.tar.gz ``` -OUTPUT=~/output; -nextflow run epi2me-labs/wf-transcriptomes \ - --fastq ERR6053095_chr20.fastq \ - --ref_genome chr20/hg38_chr20.fa \ - --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ - --jaffal_refBase chr20/ \ - --jaffal_genome hg38_chr20 \ - --jaffal_annotation "genCode22" \ - --out_dir outdir -w workspace_dir +The workflow can be run with the demo data using: ``` - -A full list of options can be seen in nextflow_schema.json. -Parameters can be specified either in a config like `parameter = value` or on the command line like `--parameter value`. -Below are some commonly used parameters in the format used in config files. - -Select how the transcriptome used for analysis should be prepared: - -- To create a reference transcriptome using an existing reference genome `--transcriptome_source reference-guided` (default) -- Use a supplied transcriptome `--transcriptome_source precomputed"` +nextflow run epi2me-labs/wf-transcriptome \ +--fastq differential_expression/differential_expression_fastq \ +--de_analysis --ref_genome differential_expression/hg38_chr20.fa \ +--transcriptome-source reference-guided \ +--ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ +--direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \ +--jaffal_refBase differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \ +-profile standard +``` +For further information about running a workflow on the cmd line see https://labs.epi2me.io/wfquickstart/ -To run the workflow with direct RNA reads `--direct_rna true` (this just skips the pychopper step). -Pychopper and minimap2 can take options via `--minimap2_opts` and `--pychopper_opts`, for example: +## Related protocols -- When using the SIRV synthetic test data - - `--minimap2_opts '-uf --splice-flank=no'` -- pychopper needs to know which cDNA synthesis kit used, which can be specified with - - SQK-PCS109: `--pychopper_opts '-k PCS109'` (default) - - SQK-PCS110: `--pychopper_opts '-k PCS110'` - - SQK-PCS111: `--pychopper_opts '-k PCS111'` - - SQK-LSK114: `--pychopper_opts '-k LSK114'` -- pychopper can use one of two available backends for identifying primers in the raw reads - - nhmmscan `--pychopper opts '-m phmm'` - - edlib `--pychopper opts '-m edlib'` +This workflow is designed to take input sequences that have been produced from [Oxford Nanopore Technologies](https://nanoporetech.com/) devices. -__Note__: edlib is set by default in the config as it's quite a lot faster. However, it may be less sensitive than nhmmscan. - -### Fusion detection - -JAFFAL from the [JAFFA](https://github.com/Oshlack/JAFFA) -package is used to identify potential fusion transcripts. - -In order to use JAFFAL, reference files must first be downloaded. -To use pre-processed hg38 genome and GENCODE v22 annotation files (as used in the JAFFAL paper) -do: -```shell -mkdir jaffal_data_dir -cd jaffal_data_dir/ -sh path/to/wf-transcriptomes/subworkflows/JAFFAL/download_jaffal_references.sh -```` -Then the path to the directory containing the downloaded reference data must be specified with -`--jaffal_refBase`. +Find related protocols in the [Nanopore community](https://community.nanoporetech.com/docs/). -**Using alternative genome and annotation files** -These should be prepared as described -[here](https://github.com/Oshlack/JAFFA/wiki/FAQandTroubleshooting#how-can-i-generate-the-reference-files-for-a-non-supported-genome). +## Inputs -The resulting JAFFAL reference files will look something like `hg38_genCode22.fa`. The following options enable JAFFAL to find these -files: +### Input Options -`--jaffal_genome reference_genome_name` optional (default: `hg38`) -`--jaffal_annotation jaffal_annotation_prefix` optional (default: `genCode22`) +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| fastq | string | FASTQ files to use in the analysis. | This accepts one of three cases: (i) the path to a single FASTQ file; (ii) the path to a top-level directory containing FASTQ files; (iii) the path to a directory containing one level of sub-directories which in turn contain FASTQ files. In the first and second case, a sample name can be supplied with `--sample`. In the last case, the data is assumed to be multiplexed with the names of the sub-directories as barcodes. In this case, a sample sheet can be provided with `--sample_sheet`. | | +| transcriptome_source | string | Select how the transcriptome used for analysis should be prepared. | To analyse only gene fusions and differential expression use of an existing transcriptome may be preferred and so 'precomputed' should be selected. In this case the 'ref_transcriptome' parameter should be specified. To create a reference transcriptome using an existing reference genome, select 'reference guided' and specify the 'ref_genome' parameter. | reference-guided | +| ref_genome | string | Path to reference genome sequence [.fa/.fq/.fa.gz/fq.gz]. Required for reference-based workflow. | A reference genome is required for reference-based assembly of a transcriptome. | | +| ref_transcriptome | string | Transcriptome reference file. Required for precomputed transcriptome calculation and for differential expression analysis. | A reference transcriptome related to the sample under study. Must be supplied when the 'Transcriptome source' parameter has been set to 'precomputed' or to perform differential expression. | | +| ref_annotation | string | A reference annotation in GFF2 or GFF3 format (extensions .gtf(.gz), .gff(.gz), .gff3(.gz)). Only annotation files from [Encode](https://www.encodeproject.org), [Ensembl](https://www.ensembl.org/index.html) and [NCBI](https://www.ncbi.nlm.nih.gov/) are supported. | This will be used for guiding the transcriptome assembly and to label transcripts with their corresponding gene identifiers. | | +| direct_rna | boolean | Set to true for direct RNA sequencing. | Omits the pychopper step. | False | +| analyse_unclassified | boolean | Analyse unclassified reads from input directory. By default the workflow will not process reads in the unclassified directory. | If selected and if the input is a multiplex directory the workflow will also process the unclassified directory. | False | -__Note__: JAFFAL is not currently working on Mac M1 (osx-arm64 architecture). +### Sample Options -### Differential Expression +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| sample_sheet | string | A CSV file used to map barcodes to sample aliases. The sample sheet can be provided when the input data is a directory containing sub-directories with FASTQ files. If you are running the differential expression workflow, there must be an additional column `condition` with two labels, one of which must be `control` (e.g. `control` and `treated`). Control will indicate which samples will be used as the reference. There should be at least 3 repeats for each condition. | The sample sheet is a CSV file with, minimally, columns named `barcode` and `alias`. Extra columns are allowed. | | +| sample | string | A single sample name for non-multiplexed data. Permissible if passing a single .fastq(.gz) file or directory of .fastq(.gz) files. | | | + + +### Options for reference-based workflow + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| plot_gffcmp_stats | boolean | Create a PDF of plots from showing gffcompare results | If set to true, a PDF file containing detailed gffcompare reults will be output | True | +| gffcompare_opts | string | Extra command-line options to give to gffcompare -r | For a list of possible options see [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml). | -R | +| minimap2_index_opts | string | Extra command-line options for minimap2 indexing. | See [minimap2 index options](https://lh3.github.io/minimap2/minimap2.html#4) for more information. These will only be relevant in the reference based transcriptome assembly. | -k14 | +| minimap2_opts | string | Additional command-line options for minimap2 alignment. | See [minimap2 options](https://lh3.github.io/minimap2/minimap2.html#5) for further information. These will only be relevant in the reference based transcriptome assembly. | -uf | +| minimum_mapping_quality | integer | filter aligned reads by MAPQ quality. | Reads that do not meet this mapping quality after minimap2 alignment, will be filtered out. | 40 | +| stringtie_opts | string | Extra command-line options for stringtie transcript assembly. | For additional String tie options see [here](https://github.com/gpertea/stringtie#stringtie-options). | --conservative | + + +### Gene Fusion Detection Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| jaffal_refBase | string | JAFFAl reference genome directory. | JAFFAL human hg38 reference data directory can be downloaded from here: https://figshare.com/ndownloader/files/25410494 or see the README for alternative instructions. If custom gemome files are required, see the instructions here: https://github.com/Oshlack/JAFFA/wiki/FAQandTroubleshooting#how-can-i-generate-the-reference-files-for-a-non-supported-genome. | | +| jaffal_genome | string | Genome reference prefix. e.g. hg38. | JAFFAL reference files are prefixed with the genome reference file name and need to be supplied . If using the human reference data provided by JAFFAL, this can be left at `hg38`. | hg38 | +| jaffal_annotation | string | Annotation suffix. | JAFFAL reference files are suffixed with the annotation filename and this needs to be supplied. For the human hg38 reference data supplied by JAFFAL, this is `genCode22`. | genCode22 | + + +### Differential Expression Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| de_analysis | boolean | Run DE anaylsis | Running this requires you to provide at least two replicates for a control and treated sample as well as a sample sheet param. | False | +| min_gene_expr | integer | Minimum gene counts | The minimum number of total mapped sequence reads for a gene to be considered expressed. | 10 | +| min_feature_expr | integer | Minimum transcript counts | The minimum number of total mapped sequence reads for a transcript to be considered. | 3 | +| min_samps_gene_expr | integer | Genes expressed in a minimum of this many samples will be included in the differential expression analysis. | A gene must be mapped to at least this minimum number of samples for the gene be included in the analysis. | 3 | +| min_samps_feature_expr | integer | Transcripts expressed in minimum this many samples | A transcript must be mapped in at least this this minimum number of samples to be included in the analysis. | 1 | + + +### Advanced Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| threads | integer | Number of CPU threads. | Only provided to processes including alignment and and assembly that benefit from multiple threads. | 4 | +| pychopper_opts | string | Extra pychopper opts | See available options (here)[https://github.com/epi2me-labs/pychopper#usage] | -m edlib | +| bundle_min_reads | integer | Minimum size of bam bundle for parallel processing. | | 50000 | +| isoform_table_nrows | integer | Maximum rows to dispay in the isoform report table | | 5000 | + + +### Miscellaneous Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| disable_ping | boolean | Enable to prevent sending a workflow ping. | | False | + + + + + + +## Outputs + +Outputs files may be aggregated including information for all samples or provided per sample. Per sample files will be prefixed with respective aliases and represented below as {{ alias }}. + +| Title | File path | Description | Per sample or aggregated | +|-------|-----------|-------------|--------------------------| +| workflow report | wf-transcriptomes-report.html | a HTML report document detailing the primary findings of the workflow | aggregated | +| Per file read stats | fastq_ingress_results/reads/fastcat_stats/per-file-stats.tsv | A TSV with per file read stats, including all samples. | aggregated | +| Per file read stats | fastq_ingress_results/reads/fastcat_stats/per-read-stats.tsv | A TSV with per read stats, including all samples. | aggregated | +| Run ID's | fastq_ingress_results/reads/fastcat_stats/run_ids | List of run IDs present in reads. | aggregated | +| Meta map json | fastq_ingress_results/reads/metamap.json | Metadata used in workflow presented in a JSON. | aggregated | +| Concatenated sequence data | fastq_ingress_results/reads/{{ alias }}.fastq.gz | Per sample reads concatenated in to one FASTQ file. | per-sample | +| Assembled transcriptome | {{ alias }}_transcriptome.fas | Per sample assembled transcriptome. | per-sample | +| Annotated assembled transcriptome | {{ alias }}_merged_transcriptome.fas | Per sample annotated assembled transcriptome. | per-sample | +| Alignment summary statistics | {{ alias }}_read_aln_stats.tsv | Per sample alignment summary statistics. | per-sample | +| GFF compare results. | {{ alias }}_gffcompare | All GFF compare output files. | per-sample | +| Differential gene expression results | /de_analysis/results_dge.tsv | This is a gene-level result file that describes genes and the probability that they show differential expression between experimental conditions . | aggregated | +| Differential gene expression report | /de_analysis/results_dge.pdf | Summary report of differential gene expression analysis as a PDF. | aggregated | +| Differential transcript usage gene TSV | /de_analysis/results_dtu_gene.tsv | This is a gene-level result file from DEXSeq that lists annotated genes and their probabilities of differential expression. | aggregated | +| Differential gene expression report | /de_analysis/results_dtu.pdf | Summary report of differential transcript usage results as a PDF. | aggregated | +| Differential transcript usage TSV | /de_analysis/results_dtu_transcript.tsv | This is a transcript-level result file from DEXSeq that lists annotated genes and their probabilities of differential expression. | aggregated | +| Differential transcript usage stageR TSV | /de_analysis/results_dtu_stageR.tsv | This is the output from StageR and it shows both gene and transcript probabilities of differential expression | aggregated | +| Differential transcript usage DEXSeq TSV | /de_analysis/results_dexseq.tsv | The complete output from the DEXSeq-analysis, shows both gene and transcript probabilities of differential expression. | aggregated | +| Gene counts | /de_analysis/all_gene_counts.tsv | Raw gene counts created by the Salmon tool, before filtering. | aggregated | +| Transcript counts | /de_analysis/all_transcript_counts.tsv | Raw transcript counts created by the Salmon tool, before filtering. | aggregated | +| Transcript counts filtered | /de_analysis/all_counts_filtered.tsv | Filtered transcript counts, used for DE_analysis. | aggregated | +| Transcript per million counts | /de_analysis/de_tpm_transcript_counts.tsv | This file shows transcript per million (TPM) of the raw counts to facilitate comparisons across sample. | aggregated | +| Final non redundant transcriptome | /de_analysis/final_non_redundant_transcriptome.fasta | Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. | aggregated | +| Fusion transcript sequences | /jaffal_output_{{ alias }}/jaffa_results.fasta | Fusion transcript sequences output by Jaffa. | per-sample | +| Fusion transcript sequence summary file | /jaffal_output_{{ alias }}/jaffa_results.csv | Fusion transcript sequences summary file output by Jaffa. | per-sample | + + + + +## Pipeline overview + +### 1. Concatenate input files and generate per read stats. +The [fastcat](https://github.com/epi2me-labs/fastcat) tool is used to concatenate multifile samples to be processed by the workflow. It will also output per read stats including average read lengths and qualities. + +### 2. Preprocess cDNA. +If input sequences are cDNA [Pychopper](https://github.com/epi2me-labs/pychopper) is used to orient, trim and rescue full length cDNA reads and associated statistics. If the `direct_rna` parameter is selected this step will be skipped. + +### 3. Build transcriptome. +If the `transcriptome_source` parameter is "reference-guided" a transcriptome will be built for each sample as outlined below. If the `transcriptome_source` is "precomputed" and the `reference_transcriptome` parameter is provided the workflow will skip step 3. + +#### 3.1 Align reads with reference genome. +The reference genome will be indexed and aligned using [Minimap2](https://github.com/lh3/minimap2). The output is sorted and converted to a BAM file using [Samtools](https://www.htslib.org/). Alignment stats are created from these using [Seqkit BAM](https://bioinf.shenwei.me/seqkit/usage/#bam). + +#### 3.2 Chunk BAM +The aligned BAMs are split into chunks using the bundle_min_reads parameter (default: 50000). + +#### 3.3 Assemble transcripts +[StringTie](https://ccb.jhu.edu/software/stringtie/) is then used to assemble the transcripts using the aligned segments in the chunked BAM files. The assembled transcript will be output as a [GFF file](https://www.ensembl.org/info/website/upload/gff3.html). If a `ref_annotation` file is provided this will also be included in the GFF. + +#### 3.4 Merge Chunks +Transcript GFF files from the chunks with the same sample aliases will then be merged. + +#### 3.5 Annnotate +[GffCompare](https://ccb.jhu.edu/software/stringtie/gffcompare.html) is then used to compare query and reference annotations, merging records where appropriate and then annotating them. This also creates estimates of accuracy of the GFF files output in a stats file per sample. + +#### 3.6 Create transcriptomes +[Gffread](https://github.com/gpertea/gffread) is used to create a transcriptome FASTA file from the final GFF as well as a merged transcriptome that includes annotations in the FASTA headers where available. + +### 4. Find gene fusions +If gene fusion options are provided, fusion gene detection is performed using [JAFFA](https://github.com/Oshlack/JAFFA), with the JAFFAL extension. To enable this provide the gene fusion detection options: `jaffal_refBase`, `jaffal_genome` and `jaffal_annotation`. + +### 5. Differential expression analysis + +Differential gene expression (DGE) and differential transcript usage (DTU) analyses aim to identify genes and transcripts that show statistically altered expression patterns. Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below. -**Example workflow for differential expression transcript assembly** - #### Sample sheet condition column The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`. - Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory). - The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files. - The condition column will need to contain one of two keys to indicate the two samples being compared. Control must be one of the keys, used to indicate which samples will be used as the reference in the differential expression analysis. -In the default `sample_sheet.csv` available in the test_data directory we have used the following. - eg. sample_sheet.csv ``` barcode,alias,condition @@ -205,74 +248,53 @@ barcode05,sample05,treated barcode06,sample06,treated ``` -You will also need to provide a reference genome and a reference annotation file. -Here is an example cmd to run the workflow. First you will need to download the data with wget. -eg. -``` -wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz -nextflow run epi2me-labs/wf-transcriptomes \ - --fastq differential_expression/differential_expression_fastq \ - --transcriptome-source precomputed \ - --de_analysis \ - --ref_genome differential_expression/hg38_chr20.fa \ - --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff \ - --direct_rna --minimap2_index_opts '-k 15' \ - --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --sample_sheet test_data/sample_sheet.csv -``` -You can also run the differential expression section of the workflow on its own by providing a reference transcriptome and setting the `--transcriptome_source` to precomputed. -eg. -``` -nextflow run epi2me-labs/wf-transcriptomes \ - --fastq differential_expression/differential_expression_fastq \ - --de_analysis \ - --transcriptome_source precomputed \ - --ref_genome differential_expression/hg38_chr20.fa \ - --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ - --direct_rna --minimap2_index_opts '-k 15' \ - --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --sample_sheet test_data/sample_sheet.csv -``` +#### 5.1 Merge cross sample transcriptomes +If a `ref_transcriptome` is not provided, the transcriptomes created by the workflow will be used for DE analysis. To do this, the GFF outputs of GffCompare are merged using StringTie. A final non redundant FASTA file of the transcripts is created using the merged GFF file and the reference genome using seqkit. -## Workflow outputs -* an HTML report document detailing the primary findings of the workflow. -* for each sample: - * [gffcomapre](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) output directories - * read_aln_stats.tsv - alignment summary statistics - * transcriptome.fas - the assembled transcriptome - * merged_transcritptome.fas - annotated, assembled transcriptome - * [jaffal](https://github.com/Oshlack/JAFFA) ooutput directories - -### Fusion detection outputs -in `${out_dir}/jaffal_output_${sample_id}` you will find: -* jaffa_results.csv - the csv results summary file -* jaffa_results.fasta - fusion transcritpt sequences +#### 5.2 Create a final non redundant transcriptome +The reads from all the samples will be aligned with the final non redundant transcriptome using Minimap2 in a splice aware manner. -### Differential Expression outputs -* `de_analysis/results_dge.tsv` and `de_analysis/results_dge.pdf`- results of `edgeR` differential gene expression analysis. -* `de_analysis/results_dtu_gene.tsv`, `de_analysis/results_dtu_transcript.tsv` and `de_analysis/results_dtu.pdf` - results of differential transcript usage by `DEXSeq`. -* `de_analysis/results_dtu_stageR.tsv` - results of the `stageR` analysis of the `DEXSeq` output. -* `de_analysis/dtu_plots.pdf` - DTU results plot based on the `stageR` results and filtered counts. -* `de_analysis/all_gene_counts.tsv` - Gene counts generated by the `salmon` tool before filtering. -* `de_analysis/de_transcript_counts.tsv` - Transcript counts generated by the `salmon` tool before filtering. -* `de_analysis/de_tpm_transcript_counts.tsv` - To facilitate comparisons across samples this file shows transcript per million (TPM) of the raw counts. -* `de_analysis/all_counts_filtered.tsv` - Transcript counts filtered with input criteria. Used for DE analysis. -* `final_non_redundant_transcriptome.fasta` - Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. +#### 5.3 Count genes and transcripts +[Salmon](https://github.com/COMBINE-lab/salmon) is used for transcript quantification, giving gene and transcript counts. -### References +#### 5.4 Pre-filtering of quantitative data using DRIMSeq +[DRIMSeq](https://bioconductor.org/packages/release/bioc/html/DRIMSeq.html) is used to filter the transcript count data from the Salmon analysis. The filter step will be used to select for genes and transcripts that satisfy rules for the number of samples in which a gene or transcript must be observed, and minimum threshold levels for the number of observed reads. The parameters used for filtering are `min_samps_gene_expr`, `min_samps_feature_expr`, `min_gene_expr`, and `min_feature_expr`. -* Benjamini, Yoav, and Yosef Hochberg. 1995. “Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing.” Journal of the Royal Statistical Society. Series B (Methodological) 57 (1): 289–300. http://www.jstor.org/stable/2346101. -* McCarthy, Davis J., Chen, Yunshun, Smyth, and Gordon K. 2012. “Differential Expression Analysis of Multifactor Rna-Seq Experiments with Respect to Biological Variation.” Nucleic Acids Research 40 (10): 4288–97. -* Nowicka, Malgorzata, and Mark D. Robinson. 2016. “DRIMSeq: A Dirichlet-Multinomial Framework for Multivariate Count Outcomes in Genomics [Version 2; Referees: 2 Approved].” F1000Research 5 (1356). https://doi.org/10.12688/f1000research.8900.2. -* Patro, Robert, Geet Duggal, Michael I Love, Rafael A Irizarry, and Carl Kingsford. 2017. “Salmon Provides Fast and Bias-Aware Quantification of Transcript Expression.” Nature Methods 14 (March). https://doi.org/10.1038/nmeth.4197. -* Robinson, Mark D, Davis J McCarthy, and Gordon K Smyth. 2010. “EdgeR: A Bioconductor Package for Differential Expression Analysis of Digital Gene Expression Data.” Bioinformatics 26 (1): 139–40. -* Love, Michael I., et al. Swimming Downstream: Statistical Analysis of Differential Transcript Usage Following Salmon Quantification. 7:952, F1000Research, 14 Sept. 2018. f1000research.com, https://f1000research.com/articles/7-952 +#### 5.5 edgeR based differential expression analysis +A statistical analysis is first performed using [edgeR](https://bioconductor.org/packages/release/bioc/html/edgeR.html) to identify the subset of differentially expressed genes. The filtered list of gene counts is used as input. A normalisation factor is calculated for each sequence library (using the default TMM method described by [McCarthy et al. (2012)](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3378882/) for further details). The defined experimental design is used to calculate estimates of dispersion for each of the gene features. Statistical tests are calculated using the contrasts defined in the experimental design. The differentially expressed genes are corrected for false discovery (fdr) using the method of Benjamini & Hochberg ([Benjamini and Hochberg (1995)](https://www.jstor.org/stable/2346101)) + +#### 5.6 Differential transcript usage using DEXSeq +Differential transcript usage analysis is performed using the R [DEXSeq](https://bioconductor.org/packages/release/bioc/html/DEXSeq.html) package ([Anders et al. (2012)](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3460195/)). Similar to the edgeR package, DEXSeq estimates the variance between the biological replicates and applies generalised linear models for the statistical testing. The key difference is that the DEXSeq method looks for differences at the exon count level. DEXSeq uses the filtered transcript count data prepared earlier in this analysis. + +#### 5.7 StageR stage-wise analysis of DGE and DTU +The final component of this isoform analysis is a stage-wise statistical test using the R software package [stageR](https://bioconductor.org/packages/release/bioc/html/stageR.html)([Van den Berge and Clement (2018)](https://genomebiology.biomedcentral.com/articles/10.1186/s13059-017-1277-0)). stageR uses (1) the raw p-values for DTU from the DEXSeq analysis in the previous section and (2) a false-discovery corrected set of p-values from testing whether individual genes contain at least one exon showing DTU. A hierarchical two-stage statistical testing evaluates the set of genes for DTU. -## Useful links -* [nextflow](https://www.nextflow.io/) -* [docker](https://www.docker.com/products/docker-desktop) -* [Singularity](https://sylabs.io/singularity/) -* [racon](https://github.com/isovic/racon) \ No newline at end of file + + + +## Troubleshooting + ++ If the workflow fails please run it with the demo data set to ensure the workflow itself is working. This will help us determine if the issue is related to the environment, input parameters or a bug. ++ See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/). + + + +## FAQ's + +*Does the workflow support de novo assembly?* - Currently the workflow does not have a *de novo* mode. + +If your question is not answered here, please report any issues or suggestions on the [github issues](https://github.com/epi2me-labs/wf-transcriptomes/issues) page or start a discussion on the [community](https://community.nanoporetech.com/). + + + +## Related blog posts + ++ [How to align your data](https://labs.epi2me.io/how-to-align/) + +See the [EPI2ME website](https://labs.epi2me.io/) for lots of other resources and blog posts. + + + diff --git a/docs/01_brief_description.md b/docs/01_brief_description.md new file mode 100644 index 0000000..bbab820 --- /dev/null +++ b/docs/01_brief_description.md @@ -0,0 +1 @@ +Transcriptome analysis including assembly and annotation of cDNA and direct RNA sequencing data, gene fusions and differential expression. \ No newline at end of file diff --git a/docs/02_introduction.md b/docs/02_introduction.md new file mode 100644 index 0000000..56ce7fa --- /dev/null +++ b/docs/02_introduction.md @@ -0,0 +1,8 @@ +This workflow can be used for the following: + ++ Identify RNA transcripts using either cDNA or direct RNA reads. ++ Reference aided transcriptome assembly. ++ Annotation of assembled transcripts. ++ Gene fusions detection. ++ Differential gene expression analysis using a pre-computed or assembled reference transcriptome. ++ Differential transcript usage analysis using a precomputed or assembled reference transcriptome. \ No newline at end of file diff --git a/docs/03_compute_requirements.md b/docs/03_compute_requirements.md new file mode 100644 index 0000000..02eafd3 --- /dev/null +++ b/docs/03_compute_requirements.md @@ -0,0 +1,13 @@ +Recommended requirements: + ++ CPUs = 16 ++ memory = 32GB + +Minimum requirement: + ++ CPUs = 8 ++ memory = 16GB + +Approximate run time: 15 minutes per sample, with 1 million reads and recommended resources. + +ARM processor support: False diff --git a/docs/04_install_and_run.md b/docs/04_install_and_run.md new file mode 100644 index 0000000..411f359 --- /dev/null +++ b/docs/04_install_and_run.md @@ -0,0 +1,34 @@ +These are instructions to install and run the workflow on command line. You can also access the workflow via the [EPI2ME application](https://labs.epi2me.io/downloads/). + +The workflow uses [Nextflow](https://www.nextflow.io/) to manage compute and software resources, therefore nextflow will need to be installed before attempting to run the workflow. + +The workflow can currently be run using either [Docker](https://www.docker.com/products/docker-desktop) or +[Singularity](https://docs.sylabs.io/guides/3.0/user-guide/index.html) to provide isolation of +the required software. Both methods are automated out-of-the-box provided +either docker or singularity is installed. This is controlled by the [`-profile`](https://www.nextflow.io/docs/latest/config.html#config-profiles) parameter as exemplified below. + +It is not required to clone or download the git repository in order to run the workflow. +More information on running EPI2ME workflows can be found on our [website](https://labs.epi2me.io/wfindex). + +The following command can be used to obtain the workflow. This will pull the repository in to the assets folder of nextflow and provide a list of all parameters available for the workflow as well as an example command: + +``` +nextflow run epi2me-labs/wf-transcriptomes -–help +``` +A demo dataset is provided for testing of the workflow. It can be downloaded using: +``` +wget https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz +tar -xzvf differential_expression.tar.gz +``` +The workflow can be run with the demo data using: +``` +nextflow run epi2me-labs/wf-transcriptome \ +--fastq differential_expression/differential_expression_fastq \ +--de_analysis --ref_genome differential_expression/hg38_chr20.fa \ +--transcriptome-source reference-guided \ +--ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ +--direct_rna --minimap2_index_opts '-k 15' --sample_sheet differential_expression/sample_sheet.csv \ +--jaffal_refBase differential_expression/chr20/ --jaffal_genome hg38_chr20 --jaffal_annotation genCode22 \ +-profile standard +``` +For further information about running a workflow on the cmd line see https://labs.epi2me.io/wfquickstart/ \ No newline at end of file diff --git a/docs/05_related_protocols.md b/docs/05_related_protocols.md new file mode 100644 index 0000000..592125f --- /dev/null +++ b/docs/05_related_protocols.md @@ -0,0 +1,3 @@ +This workflow is designed to take input sequences that have been produced from [Oxford Nanopore Technologies](https://nanoporetech.com/) devices. + +Find related protocols in the [Nanopore community](https://community.nanoporetech.com/docs/). \ No newline at end of file diff --git a/docs/06_inputs.md b/docs/06_inputs.md new file mode 100644 index 0000000..569b849 --- /dev/null +++ b/docs/06_inputs.md @@ -0,0 +1,70 @@ +### Input Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| fastq | string | FASTQ files to use in the analysis. | This accepts one of three cases: (i) the path to a single FASTQ file; (ii) the path to a top-level directory containing FASTQ files; (iii) the path to a directory containing one level of sub-directories which in turn contain FASTQ files. In the first and second case, a sample name can be supplied with `--sample`. In the last case, the data is assumed to be multiplexed with the names of the sub-directories as barcodes. In this case, a sample sheet can be provided with `--sample_sheet`. | | +| transcriptome_source | string | Select how the transcriptome used for analysis should be prepared. | To analyse only gene fusions and differential expression use of an existing transcriptome may be preferred and so 'precomputed' should be selected. In this case the 'ref_transcriptome' parameter should be specified. To create a reference transcriptome using an existing reference genome, select 'reference guided' and specify the 'ref_genome' parameter. | reference-guided | +| ref_genome | string | Path to reference genome sequence [.fa/.fq/.fa.gz/fq.gz]. Required for reference-based workflow. | A reference genome is required for reference-based assembly of a transcriptome. | | +| ref_transcriptome | string | Transcriptome reference file. Required for precomputed transcriptome calculation and for differential expression analysis. | A reference transcriptome related to the sample under study. Must be supplied when the 'Transcriptome source' parameter has been set to 'precomputed' or to perform differential expression. | | +| ref_annotation | string | A reference annotation in GFF2 or GFF3 format (extensions .gtf(.gz), .gff(.gz), .gff3(.gz)). Only annotation files from [Encode](https://www.encodeproject.org), [Ensembl](https://www.ensembl.org/index.html) and [NCBI](https://www.ncbi.nlm.nih.gov/) are supported. | This will be used for guiding the transcriptome assembly and to label transcripts with their corresponding gene identifiers. | | +| direct_rna | boolean | Set to true for direct RNA sequencing. | Omits the pychopper step. | False | +| analyse_unclassified | boolean | Analyse unclassified reads from input directory. By default the workflow will not process reads in the unclassified directory. | If selected and if the input is a multiplex directory the workflow will also process the unclassified directory. | False | + + +### Sample Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| sample_sheet | string | A CSV file used to map barcodes to sample aliases. The sample sheet can be provided when the input data is a directory containing sub-directories with FASTQ files. If you are running the differential expression workflow, there must be an additional column `condition` with two labels, one of which must be `control` (e.g. `control` and `treated`). Control will indicate which samples will be used as the reference. There should be at least 3 repeats for each condition. | The sample sheet is a CSV file with, minimally, columns named `barcode` and `alias`. Extra columns are allowed. | | +| sample | string | A single sample name for non-multiplexed data. Permissible if passing a single .fastq(.gz) file or directory of .fastq(.gz) files. | | | + + +### Options for reference-based workflow + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| plot_gffcmp_stats | boolean | Create a PDF of plots from showing gffcompare results | If set to true, a PDF file containing detailed gffcompare reults will be output | True | +| gffcompare_opts | string | Extra command-line options to give to gffcompare -r | For a list of possible options see [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml). | -R | +| minimap2_index_opts | string | Extra command-line options for minimap2 indexing. | See [minimap2 index options](https://lh3.github.io/minimap2/minimap2.html#4) for more information. These will only be relevant in the reference based transcriptome assembly. | -k14 | +| minimap2_opts | string | Additional command-line options for minimap2 alignment. | See [minimap2 options](https://lh3.github.io/minimap2/minimap2.html#5) for further information. These will only be relevant in the reference based transcriptome assembly. | -uf | +| minimum_mapping_quality | integer | filter aligned reads by MAPQ quality. | Reads that do not meet this mapping quality after minimap2 alignment, will be filtered out. | 40 | +| stringtie_opts | string | Extra command-line options for stringtie transcript assembly. | For additional String tie options see [here](https://github.com/gpertea/stringtie#stringtie-options). | --conservative | + + +### Gene Fusion Detection Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| jaffal_refBase | string | JAFFAl reference genome directory. | JAFFAL human hg38 reference data directory can be downloaded from here: https://figshare.com/ndownloader/files/25410494 or see the README for alternative instructions. If custom gemome files are required, see the instructions here: https://github.com/Oshlack/JAFFA/wiki/FAQandTroubleshooting#how-can-i-generate-the-reference-files-for-a-non-supported-genome. | | +| jaffal_genome | string | Genome reference prefix. e.g. hg38. | JAFFAL reference files are prefixed with the genome reference file name and need to be supplied . If using the human reference data provided by JAFFAL, this can be left at `hg38`. | hg38 | +| jaffal_annotation | string | Annotation suffix. | JAFFAL reference files are suffixed with the annotation filename and this needs to be supplied. For the human hg38 reference data supplied by JAFFAL, this is `genCode22`. | genCode22 | + + +### Differential Expression Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| de_analysis | boolean | Run DE anaylsis | Running this requires you to provide at least two replicates for a control and treated sample as well as a sample sheet param. | False | +| min_gene_expr | integer | Minimum gene counts | The minimum number of total mapped sequence reads for a gene to be considered expressed. | 10 | +| min_feature_expr | integer | Minimum transcript counts | The minimum number of total mapped sequence reads for a transcript to be considered. | 3 | +| min_samps_gene_expr | integer | Genes expressed in a minimum of this many samples will be included in the differential expression analysis. | A gene must be mapped to at least this minimum number of samples for the gene be included in the analysis. | 3 | +| min_samps_feature_expr | integer | Transcripts expressed in minimum this many samples | A transcript must be mapped in at least this this minimum number of samples to be included in the analysis. | 1 | + + +### Advanced Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| threads | integer | Number of CPU threads. | Only provided to processes including alignment and and assembly that benefit from multiple threads. | 4 | +| pychopper_opts | string | Extra pychopper opts | See available options (here)[https://github.com/epi2me-labs/pychopper#usage] | -m edlib | +| bundle_min_reads | integer | Minimum size of bam bundle for parallel processing. | | 50000 | +| isoform_table_nrows | integer | Maximum rows to dispay in the isoform report table | | 5000 | + + +### Miscellaneous Options + +| Nextflow parameter name | Type | Description | Help | Default | +|--------------------------|------|-------------|------|---------| +| disable_ping | boolean | Enable to prevent sending a workflow ping. | | False | + + diff --git a/docs/07_outputs.md b/docs/07_outputs.md new file mode 100644 index 0000000..e7854fd --- /dev/null +++ b/docs/07_outputs.md @@ -0,0 +1,28 @@ +Outputs files may be aggregated including information for all samples or provided per sample. Per sample files will be prefixed with respective aliases and represented below as {{ alias }}. + +| Title | File path | Description | Per sample or aggregated | +|-------|-----------|-------------|--------------------------| +| workflow report | wf-transcriptomes-report.html | a HTML report document detailing the primary findings of the workflow | aggregated | +| Per file read stats | fastq_ingress_results/reads/fastcat_stats/per-file-stats.tsv | A TSV with per file read stats, including all samples. | aggregated | +| Per file read stats | fastq_ingress_results/reads/fastcat_stats/per-read-stats.tsv | A TSV with per read stats, including all samples. | aggregated | +| Run ID's | fastq_ingress_results/reads/fastcat_stats/run_ids | List of run IDs present in reads. | aggregated | +| Meta map json | fastq_ingress_results/reads/metamap.json | Metadata used in workflow presented in a JSON. | aggregated | +| Concatenated sequence data | fastq_ingress_results/reads/{{ alias }}.fastq.gz | Per sample reads concatenated in to one FASTQ file. | per-sample | +| Assembled transcriptome | {{ alias }}_transcriptome.fas | Per sample assembled transcriptome. | per-sample | +| Annotated assembled transcriptome | {{ alias }}_merged_transcriptome.fas | Per sample annotated assembled transcriptome. | per-sample | +| Alignment summary statistics | {{ alias }}_read_aln_stats.tsv | Per sample alignment summary statistics. | per-sample | +| GFF compare results. | {{ alias }}_gffcompare | All GFF compare output files. | per-sample | +| Differential gene expression results | /de_analysis/results_dge.tsv | This is a gene-level result file that describes genes and the probability that they show differential expression between experimental conditions . | aggregated | +| Differential gene expression report | /de_analysis/results_dge.pdf | Summary report of differential gene expression analysis as a PDF. | aggregated | +| Differential transcript usage gene TSV | /de_analysis/results_dtu_gene.tsv | This is a gene-level result file from DEXSeq that lists annotated genes and their probabilities of differential expression. | aggregated | +| Differential gene expression report | /de_analysis/results_dtu.pdf | Summary report of differential transcript usage results as a PDF. | aggregated | +| Differential transcript usage TSV | /de_analysis/results_dtu_transcript.tsv | This is a transcript-level result file from DEXSeq that lists annotated genes and their probabilities of differential expression. | aggregated | +| Differential transcript usage stageR TSV | /de_analysis/results_dtu_stageR.tsv | This is the output from StageR and it shows both gene and transcript probabilities of differential expression | aggregated | +| Differential transcript usage DEXSeq TSV | /de_analysis/results_dexseq.tsv | The complete output from the DEXSeq-analysis, shows both gene and transcript probabilities of differential expression. | aggregated | +| Gene counts | /de_analysis/all_gene_counts.tsv | Raw gene counts created by the Salmon tool, before filtering. | aggregated | +| Transcript counts | /de_analysis/all_transcript_counts.tsv | Raw transcript counts created by the Salmon tool, before filtering. | aggregated | +| Transcript counts filtered | /de_analysis/all_counts_filtered.tsv | Filtered transcript counts, used for DE_analysis. | aggregated | +| Transcript per million counts | /de_analysis/de_tpm_transcript_counts.tsv | This file shows transcript per million (TPM) of the raw counts to facilitate comparisons across sample. | aggregated | +| Final non redundant transcriptome | /de_analysis/final_non_redundant_transcriptome.fasta | Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. | aggregated | +| Fusion transcript sequences | /jaffal_output_{{ alias }}/jaffa_results.fasta | Fusion transcript sequences output by Jaffa. | per-sample | +| Fusion transcript sequence summary file | /jaffal_output_{{ alias }}/jaffa_results.csv | Fusion transcript sequences summary file output by Jaffa. | per-sample | diff --git a/docs/08_pipeline_overview.md b/docs/08_pipeline_overview.md new file mode 100644 index 0000000..f6cc24f --- /dev/null +++ b/docs/08_pipeline_overview.md @@ -0,0 +1,76 @@ +### 1. Concatenate input files and generate per read stats. +The [fastcat](https://github.com/epi2me-labs/fastcat) tool is used to concatenate multifile samples to be processed by the workflow. It will also output per read stats including average read lengths and qualities. + +### 2. Preprocess cDNA. +If input sequences are cDNA [Pychopper](https://github.com/epi2me-labs/pychopper) is used to orient, trim and rescue full length cDNA reads and associated statistics. If the `direct_rna` parameter is selected this step will be skipped. + +### 3. Build transcriptome. +If the `transcriptome_source` parameter is "reference-guided" a transcriptome will be built for each sample as outlined below. If the `transcriptome_source` is "precomputed" and the `reference_transcriptome` parameter is provided the workflow will skip step 3. + +#### 3.1 Align reads with reference genome. +The reference genome will be indexed and aligned using [Minimap2](https://github.com/lh3/minimap2). The output is sorted and converted to a BAM file using [Samtools](https://www.htslib.org/). Alignment stats are created from these using [Seqkit BAM](https://bioinf.shenwei.me/seqkit/usage/#bam). + +#### 3.2 Chunk BAM +The aligned BAMs are split into chunks using the bundle_min_reads parameter (default: 50000). + +#### 3.3 Assemble transcripts +[StringTie](https://ccb.jhu.edu/software/stringtie/) is then used to assemble the transcripts using the aligned segments in the chunked BAM files. The assembled transcript will be output as a [GFF file](https://www.ensembl.org/info/website/upload/gff3.html). If a `ref_annotation` file is provided this will also be included in the GFF. + +#### 3.4 Merge Chunks +Transcript GFF files from the chunks with the same sample aliases will then be merged. + +#### 3.5 Annnotate +[GffCompare](https://ccb.jhu.edu/software/stringtie/gffcompare.html) is then used to compare query and reference annotations, merging records where appropriate and then annotating them. This also creates estimates of accuracy of the GFF files output in a stats file per sample. + +#### 3.6 Create transcriptomes +[Gffread](https://github.com/gpertea/gffread) is used to create a transcriptome FASTA file from the final GFF as well as a merged transcriptome that includes annotations in the FASTA headers where available. + +### 4. Find gene fusions +If gene fusion options are provided, fusion gene detection is performed using [JAFFA](https://github.com/Oshlack/JAFFA), with the JAFFAL extension. To enable this provide the gene fusion detection options: `jaffal_refBase`, `jaffal_genome` and `jaffal_annotation`. + +### 5. Differential expression analysis + +Differential gene expression (DGE) and differential transcript usage (DTU) analyses aim to identify genes and transcripts that show statistically altered expression patterns. + +Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below. + +#### Sample sheet condition column +The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`. +- Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory). +- The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files. +- The condition column will need to contain one of two keys to indicate the two samples being compared. Control must be one of the keys, used to indicate which samples will be used as the reference in the differential expression analysis. + +eg. sample_sheet.csv +``` +barcode,alias,condition +barcode01,sample01,control +barcode02,sample02,control +barcode03,sample03,control +barcode04,sample04,treated +barcode05,sample05,treated +barcode06,sample06,treated +``` + +#### 5.1 Merge cross sample transcriptomes +If a `ref_transcriptome` is not provided, the transcriptomes created by the workflow will be used for DE analysis. To do this, the GFF outputs of GffCompare are merged using StringTie. A final non redundant FASTA file of the transcripts is created using the merged GFF file and the reference genome using seqkit. + +#### 5.2 Create a final non redundant transcriptome +The reads from all the samples will be aligned with the final non redundant transcriptome using Minimap2 in a splice aware manner. + +#### 5.3 Count genes and transcripts +[Salmon](https://github.com/COMBINE-lab/salmon) is used for transcript quantification, giving gene and transcript counts. + +#### 5.4 Pre-filtering of quantitative data using DRIMSeq +[DRIMSeq](https://bioconductor.org/packages/release/bioc/html/DRIMSeq.html) is used to filter the transcript count data from the Salmon analysis. The filter step will be used to select for genes and transcripts that satisfy rules for the number of samples in which a gene or transcript must be observed, and minimum threshold levels for the number of observed reads. The parameters used for filtering are `min_samps_gene_expr`, `min_samps_feature_expr`, `min_gene_expr`, and `min_feature_expr`. + +#### 5.5 edgeR based differential expression analysis +A statistical analysis is first performed using [edgeR](https://bioconductor.org/packages/release/bioc/html/edgeR.html) to identify the subset of differentially expressed genes. The filtered list of gene counts is used as input. A normalisation factor is calculated for each sequence library (using the default TMM method described by [McCarthy et al. (2012)](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3378882/) for further details). The defined experimental design is used to calculate estimates of dispersion for each of the gene features. Statistical tests are calculated using the contrasts defined in the experimental design. The differentially expressed genes are corrected for false discovery (fdr) using the method of Benjamini & Hochberg ([Benjamini and Hochberg (1995)](https://www.jstor.org/stable/2346101)) + +#### 5.6 Differential transcript usage using DEXSeq +Differential transcript usage analysis is performed using the R [DEXSeq](https://bioconductor.org/packages/release/bioc/html/DEXSeq.html) package ([Anders et al. (2012)](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3460195/)). Similar to the edgeR package, DEXSeq estimates the variance between the biological replicates and applies generalised linear models for the statistical testing. The key difference is that the DEXSeq method looks for differences at the exon count level. DEXSeq uses the filtered transcript count data prepared earlier in this analysis. + +#### 5.7 StageR stage-wise analysis of DGE and DTU +The final component of this isoform analysis is a stage-wise statistical test using the R software package [stageR](https://bioconductor.org/packages/release/bioc/html/stageR.html)([Van den Berge and Clement (2018)](https://genomebiology.biomedcentral.com/articles/10.1186/s13059-017-1277-0)). stageR uses (1) the raw p-values for DTU from the DEXSeq analysis in the previous section and (2) a false-discovery corrected set of p-values from testing whether individual genes contain at least one exon showing DTU. A hierarchical two-stage statistical testing evaluates the set of genes for DTU. + + + diff --git a/docs/09_troubleshooting.md b/docs/09_troubleshooting.md new file mode 100644 index 0000000..ecb81f2 --- /dev/null +++ b/docs/09_troubleshooting.md @@ -0,0 +1,2 @@ ++ If the workflow fails please run it with the demo data set to ensure the workflow itself is working. This will help us determine if the issue is related to the environment, input parameters or a bug. ++ See how to interpret some common nextflow exit codes [here](https://labs.epi2me.io/trouble-shooting/). \ No newline at end of file diff --git a/docs/10_FAQ.md b/docs/10_FAQ.md new file mode 100644 index 0000000..7d65676 --- /dev/null +++ b/docs/10_FAQ.md @@ -0,0 +1,3 @@ +*Does the workflow support de novo assembly?* - Currently the workflow does not have a *de novo* mode. + +If your question is not answered here, please report any issues or suggestions on the [github issues](https://github.com/epi2me-labs/wf-transcriptomes/issues) page or start a discussion on the [community](https://community.nanoporetech.com/). \ No newline at end of file diff --git a/docs/11_other.md b/docs/11_other.md new file mode 100644 index 0000000..dfbe7e2 --- /dev/null +++ b/docs/11_other.md @@ -0,0 +1,3 @@ ++ [How to align your data](https://labs.epi2me.io/how-to-align/) + +See the [EPI2ME website](https://labs.epi2me.io/) for lots of other resources and blog posts. \ No newline at end of file diff --git a/docs/header.md b/docs/header.md deleted file mode 100644 index e4bbb42..0000000 --- a/docs/header.md +++ /dev/null @@ -1,4 +0,0 @@ -# wf-transcriptomes - -This repository contains a [nextflow](https://www.nextflow.io/) workflow -for assembly and annotation of transcripts from Oxford Nanopore cDNA or direct RNA reads. \ No newline at end of file diff --git a/docs/intro.md b/docs/intro.md deleted file mode 100644 index 24e6152..0000000 --- a/docs/intro.md +++ /dev/null @@ -1,64 +0,0 @@ -## Introduction - -This workflow identifies RNA isoforms using either cDNA or direct RNA (dRNA) -Oxford Nanopore reads. - -### Preprocesing -cDNA reads are initially preprocessed by [pychopper](https://github.com/epi2me-labs/pychopper) -for the identification of full-length reads, as well as trimming and orientation correction (This step is omitted for - direct RNA reads). - - -### Transcript assembly - -#### Reference-aided transcript assembly approach -* Full length reads are mapped to a supplied reference genome using [minimap2](https://github.com/lh3/minimap2) -* Transcripts are assembled by [stringtie](http://ccb.jhu.edu/software/stringtie) -in long read mode (with or without a guide reference annotation) to generate the GFF annotation. -* The annotation generated by the pipeline is compared to the reference annotation. -using [gffcompare](http://ccb.jhu.edu/software/stringtie/gffcompare.shtml) - -### Fusion gene detection -Fusion gene detection is performed using [JAFFA](https://github.com/Oshlack/JAFFA), with the JAFFAL extension for use -with ONT long reads. - -### Differential expression analysis - -Differential gene expression (DGE) and differential transcript usage (DTU) analyses aim to identify genes and/or transcripts that show statistically altered expression patterns in a studied biological system. The results of the differential analyses are presented in a quantitative format and therefore the degree of change (up or down regulation) between experimental conditions can be calculated for each gene identified. - -These differential analyses work by taking a “snapshot” of mRNA abundance and calculating the relative levels of transcripts and isoforms. In this context, expression corresponds to the number of messenger RNAs (mRNA) measured from each gene isoform within the organism / tissue / culture being investigated. In order to determine expression levels across the whole genome, sequence data specifically targeting the mRNA molecules can be generated. - -Oxford Nanopore Technologies provides a number of sequencing solutions to allow users to generate the required snapshot of gene expression. This can be achieved by both sequencing the mRNA directly, or via a complementary DNA (cDNA) proxy. In contrast to short read sequencing technologies, entire mRNA transcripts can be captured as single reads. The example data provided with this tutorial is from a study based on the PCR-cDNA kit. This is a robust choice for performing differential transcript usage studies. This kit is suitable for preparation of sequence libraries from low mRNA input quantities. The cDNA population is enriched through PCR with low bias; an important prerequisite for the subsequent statistical analysis. - -[Workflow-transcriptomes](https://github.com/epi2me-labs/wf-transcriptomes) includes a subworkflow for DGE and DTU. The first step involves using either a reference alignment or _de novo_ assembly approach to create a set of mRNA sequences per sample. These are merged into a non-redundant transcriptome using [stringtie merge](http://ccb.jhu.edu/software/stringtie). The reads are then aligned to the transcriptome using minimap2 in a splice-aware manner. [Salmon](https://github.com/COMBINE-lab/salmon) is used for transcript quantification, giving per transcript counts and then the following R packages are used for analysis. - -### Pre-filtering of quantitative data using DRIMSeq -DRIMSeq (Nowicka and Robinson (2016)) is used to filter the transcript count data from the salmon analysis. The filter step will be used to select for genes and transcripts that satisfy rules for the number of samples in which a gene or transcript must be observed and minimum threshold levels for the number of observed reads. The parameters used for filtering are defined in the config.yaml file. The default parameters defined for this analysis include -* min_samps_gene_expr = 3 - a transcript must be mapped to a gene in at least this minimum number of samples for the gene be included in the analysis -* min_samps_feature_expr = 1 - a transcript must be mapped to an isoform in at least this this minimum number of samples for the gene isoform to be included in the analysis -* min_gene_expr = 10 - the minimum number of total mapped sequence reads for a gene to be considered expressed -* min_feature_expr = 3 - the minimum number of total mapped sequence reads for a gene isoform to be considered - -### edgeR based differential expression analysis -+A statistical analysis is first performed using edgeR (Robinson, McCarthy, and Smyth (2010), McCarthy et al. (2012)) to identify the subset of differentially expressed genes. The filtered list of gene counts is used as input. A normalisation factor is calculated for each sequence library (using the default TMM method - please see McCarthy et al. (2012) for further details). The defined experimental design is used to calculate estimates of dispersion for each of the gene features. Statistical tests are calculated using the contrasts defined in the experimental design. The differentially expressed genes are corrected for false discovery (fdr) using the method of Benjamini & Hochberg (Benjamini and Hochberg (1995)) - -### Differential transcript usage using DEXSeq -Differential transcript usage analysis is performed using the R DEXSeq package (Reyes et al. (2013)). Similar to the edgeR package, DEXSeq estimates the variance between the biological replicates and applies generalised linear models for the statistical testing. The key difference is that the DEXSeq method looks for differences at the exon count level. DEXSeq uses the filtered transcript count data prepared earlier in this analysis. - -### StageR stage-wise analysis of DGE and DTU -The final component of this isoform analysis is a stage-wise statistical test using the R software package `stageR` (Van den Berge and Clement (2018)). stageR uses (1) the raw p-values for DTU from the DEXSeq analysis in the previous section and (2) a false-discovery corrected set of p-values from testing whether individual genes contain at least one exon showing DTU. A hierarchical two-stage statistical testing evaluates the set of genes for DTU. - -## Running the workflow -For the differential expression analysis section you should have at least 3 repeats for each sample. -Your FASTQ data will need to be organised in to 6 directories that represent 3 repeats for each condition. - - -## Analysis -Differential gene expression is sensitive to the input data quantity and quality. There should be equivalence between samples in the number of sequence reads, mapped reads and quality scores. The sequence and alignment summary plots in the report can be used to assess these metrics. There is also a table that shows the transcript per million(TPM) calculated from the salmon counts. TPM normalizes the data for gene length and then sequencing depth, and makes it easier to compare across samples compared to counts. - -### Workflow inputs -- Directory containing cDNA/direct RNA reads. Or a directory containing subdirectories each with reads from different samples - (in fastq/fastq.gz format) -- Reference genome in fasta format (required for reference-based assembly). -- Optional reference annotation in GFF2/3 format (extensions allowed are .gtf(.gz), .gff(.gz), .gff3(.gz)) (required for differential expression analysis `--de_analysis`). Only annotation files from [Encode](https://www.encodeproject.org), [Ensembl](https://www.ensembl.org/index.html) and [NCBI](https://www.ncbi.nlm.nih.gov/) are supported. -- For fusion detection, JAFFAL reference files (see Quickstart) diff --git a/docs/links.md b/docs/links.md deleted file mode 100644 index 0a71673..0000000 --- a/docs/links.md +++ /dev/null @@ -1,6 +0,0 @@ -## Useful links - -* [nextflow](https://www.nextflow.io/) -* [docker](https://www.docker.com/products/docker-desktop) -* [Singularity](https://sylabs.io/singularity/) -* [racon](https://github.com/isovic/racon) \ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md deleted file mode 100644 index e1530d9..0000000 --- a/docs/quickstart.md +++ /dev/null @@ -1,194 +0,0 @@ -## Quickstart - -The workflow uses [nextflow](https://www.nextflow.io/) to manage compute and -software resources, as such nextflow will need to be installed before attempting -to run the workflow. - -The workflow can currently be run using either -[Docker](https://www.docker.com/products/docker-desktop), -[Singularity](https://sylabs.io/singularity/) to provide isolation of -the required software. Each method is automated out-of-the-box provided -either docker or singularity is installed. - -It is not required to clone or download the git repository in order to run the workflow. -For more information on running EPI2ME Labs workflows [visit out website](https://labs.epi2me.io/wfindex). - - -### Workflow options - -To obtain the workflow, having installed `nextflow`, users can run: - -``` -nextflow run epi2me-labs/wf-transcriptomes --help -``` - -to see the options for the workflow. - -**Download demonstration data** - -A small test dataset is provided for the purposes of testing the workflow software. It consists of reads, reference, -and annotations from human chromosome 20 only. -It can be downloaded using: -```shell -wget -O test_data.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/wf-isoforms_test_data.tar.gz -tar -xzvf test_data.tar.gz -``` - -**Example execution of a workflow for reference-based transcript assembly and fusion detection** -``` -OUTPUT=~/output; -nextflow run epi2me-labs/wf-transcriptomes \ - --fastq ERR6053095_chr20.fastq \ - --ref_genome chr20/hg38_chr20.fa \ - --ref_annotation chr20/gencode.v22.annotation.chr20.gtf \ - --jaffal_refBase chr20/ \ - --jaffal_genome hg38_chr20 \ - --jaffal_annotation "genCode22" \ - --out_dir outdir -w workspace_dir -``` - -A full list of options can be seen in nextflow_schema.json. -Parameters can be specified either in a config like `parameter = value` or on the command line like `--parameter value`. -Below are some commonly used parameters in the format used in config files. - -Select how the transcriptome used for analysis should be prepared: - -- To create a reference transcriptome using an existing reference genome `--transcriptome_source reference-guided` (default) -- Use a supplied transcriptome `--transcriptome_source precomputed"` - - -To run the workflow with direct RNA reads `--direct_rna true` (this just skips the pychopper step). - -Pychopper and minimap2 can take options via `--minimap2_opts` and `--pychopper_opts`, for example: - -- When using the SIRV synthetic test data - - `--minimap2_opts '-uf --splice-flank=no'` -- pychopper needs to know which cDNA synthesis kit used, which can be specified with - - SQK-PCS109: `--pychopper_opts '-k PCS109'` (default) - - SQK-PCS110: `--pychopper_opts '-k PCS110'` - - SQK-PCS111: `--pychopper_opts '-k PCS111'` - - SQK-LSK114: `--pychopper_opts '-k LSK114'` -- pychopper can use one of two available backends for identifying primers in the raw reads - - nhmmscan `--pychopper opts '-m phmm'` - - edlib `--pychopper opts '-m edlib'` - -__Note__: edlib is set by default in the config as it's quite a lot faster. However, it may be less sensitive than nhmmscan. - -### Fusion detection - -JAFFAL from the [JAFFA](https://github.com/Oshlack/JAFFA) -package is used to identify potential fusion transcripts. - -In order to use JAFFAL, reference files must first be downloaded. -To use pre-processed hg38 genome and GENCODE v22 annotation files (as used in the JAFFAL paper) -do: -```shell -mkdir jaffal_data_dir -cd jaffal_data_dir/ -sh path/to/wf-transcriptomes/subworkflows/JAFFAL/download_jaffal_references.sh -```` -Then the path to the directory containing the downloaded reference data must be specified with -`--jaffal_refBase`. - - -**Using alternative genome and annotation files** - -These should be prepared as described -[here](https://github.com/Oshlack/JAFFA/wiki/FAQandTroubleshooting#how-can-i-generate-the-reference-files-for-a-non-supported-genome). - -The resulting JAFFAL reference files will look something like `hg38_genCode22.fa`. The following options enable JAFFAL to find these -files: - -`--jaffal_genome reference_genome_name` optional (default: `hg38`) -`--jaffal_annotation jaffal_annotation_prefix` optional (default: `genCode22`) - - -__Note__: JAFFAL is not currently working on Mac M1 (osx-arm64 architecture). - -### Differential Expression - -Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below. - -**Example workflow for differential expression transcript assembly** - -#### Sample sheet condition column -The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`. -- Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory). -- The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files. -- The condition column will need to contain one of two keys to indicate the two samples being compared. Control must be one of the keys, used to indicate which samples will be used as the reference in the differential expression analysis. - -In the default `sample_sheet.csv` available in the test_data directory we have used the following. - -eg. sample_sheet.csv -``` -barcode,alias,condition -barcode01,sample01,control -barcode02,sample02,control -barcode03,sample03,control -barcode04,sample04,treated -barcode05,sample05,treated -barcode06,sample06,treated -``` - -You will also need to provide a reference genome and a reference annotation file. -Here is an example cmd to run the workflow. First you will need to download the data with wget. -eg. -``` -wget -O differential_expression.tar.gz https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-isoforms/differential_expression.tar.gz && tar -xzvf differential_expression.tar.gz -nextflow run epi2me-labs/wf-transcriptomes \ - --fastq differential_expression/differential_expression_fastq \ - --transcriptome-source precomputed \ - --de_analysis \ - --ref_genome differential_expression/hg38_chr20.fa \ - --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff \ - --direct_rna --minimap2_index_opts '-k 15' \ - --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --sample_sheet test_data/sample_sheet.csv -``` -You can also run the differential expression section of the workflow on its own by providing a reference transcriptome and setting the `--transcriptome_source` to precomputed. -eg. -``` -nextflow run epi2me-labs/wf-transcriptomes \ - --fastq differential_expression/differential_expression_fastq \ - --de_analysis \ - --transcriptome_source precomputed \ - --ref_genome differential_expression/hg38_chr20.fa \ - --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ - --direct_rna --minimap2_index_opts '-k 15' \ - --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --sample_sheet test_data/sample_sheet.csv -``` - -## Workflow outputs -* an HTML report document detailing the primary findings of the workflow. -* for each sample: - * [gffcomapre](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) output directories - * read_aln_stats.tsv - alignment summary statistics - * transcriptome.fas - the assembled transcriptome - * merged_transcritptome.fas - annotated, assembled transcriptome - * [jaffal](https://github.com/Oshlack/JAFFA) ooutput directories - -### Fusion detection outputs -in `${out_dir}/jaffal_output_${sample_id}` you will find: -* jaffa_results.csv - the csv results summary file -* jaffa_results.fasta - fusion transcritpt sequences - -### Differential Expression outputs -* `de_analysis/results_dge.tsv` and `de_analysis/results_dge.pdf`- results of `edgeR` differential gene expression analysis. -* `de_analysis/results_dtu_gene.tsv`, `de_analysis/results_dtu_transcript.tsv` and `de_analysis/results_dtu.pdf` - results of differential transcript usage by `DEXSeq`. -* `de_analysis/results_dtu_stageR.tsv` - results of the `stageR` analysis of the `DEXSeq` output. -* `de_analysis/dtu_plots.pdf` - DTU results plot based on the `stageR` results and filtered counts. -* `de_analysis/all_gene_counts.tsv` - Gene counts generated by the `salmon` tool before filtering. -* `de_analysis/de_transcript_counts.tsv` - Transcript counts generated by the `salmon` tool before filtering. -* `de_analysis/de_tpm_transcript_counts.tsv` - To facilitate comparisons across samples this file shows transcript per million (TPM) of the raw counts. -* `de_analysis/all_counts_filtered.tsv` - Transcript counts filtered with input criteria. Used for DE analysis. -* `final_non_redundant_transcriptome.fasta` - Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis. - -### References - -* Benjamini, Yoav, and Yosef Hochberg. 1995. “Controlling the False Discovery Rate: A Practical and Powerful Approach to Multiple Testing.” Journal of the Royal Statistical Society. Series B (Methodological) 57 (1): 289–300. http://www.jstor.org/stable/2346101. -* McCarthy, Davis J., Chen, Yunshun, Smyth, and Gordon K. 2012. “Differential Expression Analysis of Multifactor Rna-Seq Experiments with Respect to Biological Variation.” Nucleic Acids Research 40 (10): 4288–97. -* Nowicka, Malgorzata, and Mark D. Robinson. 2016. “DRIMSeq: A Dirichlet-Multinomial Framework for Multivariate Count Outcomes in Genomics [Version 2; Referees: 2 Approved].” F1000Research 5 (1356). https://doi.org/10.12688/f1000research.8900.2. -* Patro, Robert, Geet Duggal, Michael I Love, Rafael A Irizarry, and Carl Kingsford. 2017. “Salmon Provides Fast and Bias-Aware Quantification of Transcript Expression.” Nature Methods 14 (March). https://doi.org/10.1038/nmeth.4197. -* Robinson, Mark D, Davis J McCarthy, and Gordon K Smyth. 2010. “EdgeR: A Bioconductor Package for Differential Expression Analysis of Digital Gene Expression Data.” Bioinformatics 26 (1): 139–40. -* Love, Michael I., et al. Swimming Downstream: Statistical Analysis of Differential Transcript Usage Following Salmon Quantification. 7:952, F1000Research, 14 Sept. 2018. f1000research.com, https://f1000research.com/articles/7-952 \ No newline at end of file diff --git a/main.nf b/main.nf index 76f128d..eacecde 100644 --- a/main.nf +++ b/main.nf @@ -19,6 +19,7 @@ OPTIONAL_FILE = file("$projectDir/data/OPTIONAL_FILE") process getVersions { label "isoforms" cpus 1 + memory "500MB" output: path "versions.txt" script: @@ -43,6 +44,7 @@ process getVersions { process getParams { label "isoforms" cpus 1 + memory "500MB" output: path "params.json" script: @@ -58,6 +60,7 @@ process getParams { process decompress_ref { label "isoforms" cpus 1 + memory "500MB" input: path compressed_ref output: @@ -71,6 +74,7 @@ process decompress_ref { process decompress_annotation { label "isoforms" cpus 1 + memory "500MB" input: path compressed_annotation output: @@ -85,6 +89,7 @@ process decompress_annotation { process decompress_transcriptome { label "isoforms" cpus 1 + memory "500MB" input: path "compressed_ref.gz" output: @@ -99,6 +104,7 @@ process decompress_transcriptome { process preprocess_ref_annotation { label "isoforms" cpus 1 + memory "500MB" input: path ref_annotation output: @@ -113,6 +119,7 @@ process preprocess_ref_annotation { process preprocess_ref_transcriptome { label "isoforms" cpus 1 + memory "500MB" input: path "ref_transcriptome" output: @@ -133,7 +140,7 @@ process preprocess_reads { label "isoforms" cpus 4 - + memory "2 GB" input: tuple val(meta), path(input_reads) output: @@ -157,6 +164,7 @@ process build_minimap_index{ */ label "isoforms" cpus params.threads + memory "16 GB" input: path reference @@ -178,6 +186,7 @@ process split_bam{ label 'isoforms' cpus params.threads + memory "2 GB" input: tuple val(sample_id), path(bam) @@ -223,6 +232,7 @@ process assemble_transcripts{ */ label 'isoforms' cpus params.threads + memory "2 GB" input: tuple val(sample_id), path(bam), path(ref_annotation) @@ -245,6 +255,8 @@ process merge_gff_bundles{ Merge gff bundles into a single gff file per sample. */ label 'isoforms' + cpus params.threads + memory "2 GB" input: tuple val(sample_id), path (gff_bundle) @@ -272,7 +284,8 @@ process run_gffcompare{ */ label 'isoforms' - + cpus 1 + memory "2 GB" input: tuple val(sample_id), path(query_annotation) path ref_annotation @@ -304,7 +317,8 @@ process get_transcriptome{ Write out a transcriptome file based on the query gff annotations. */ label 'isoforms' - + cpus 1 + memory "2 GB" input: tuple val(sample_id), path(transcripts_gff), path(gffcmp_dir), path(reference_seq) output: @@ -325,6 +339,8 @@ process get_transcriptome{ process merge_transcriptomes { // Merge the transcriptomes from all samples label 'isoforms' + cpus 2 + memory "4 GB" input: path "query_annotations/*" path ref_annotation @@ -348,6 +364,8 @@ process merge_transcriptomes { process makeReport { label "isoforms" + cpus 2 + memory "2 GB" input: path versions @@ -417,6 +435,8 @@ process makeReport { // into it. process collectFastqIngressResultsInDir { label "isoforms" + cpus 1 + memory "2 GB" input: // both the fastcat seqs as well as stats might be `OPTIONAL_FILE` --> stage in // different sub-directories to avoid name collisions @@ -448,6 +468,8 @@ process collectFastqIngressResultsInDir { process output { // publish inputs to output directory label "isoforms" + cpus 1 + memory "2 GB" publishDir ( params.out_dir, mode: "copy", diff --git a/nextflow.config b/nextflow.config index a039050..c970822 100644 --- a/nextflow.config +++ b/nextflow.config @@ -90,12 +90,12 @@ params { wf { example_cmd = [ - "--condition_sheet 'wf-transcriptomes-demo/condition_sheet.tsv'", - "--direct_rna", - "--fastq 'wf-transcriptomes-demo/differential_expression_fastq'", - "--minimap2_index_opts '-k15'", - "--ref_annotation 'wf-transcriptomes-demo/gencode.v22.annotation.chr20.gtf'", - "--ref_genome 'wf-transcriptomes-demo/hg38_chr20.fa'", + "--fastq 'differential_expression/differential_expression_fastq'", + "--de_analysis --ref_genome 'differential_expression/hg38_chr20.fa'" , + "--transcriptome-source 'reference-guided'", + "--ref_annotation 'differential_expression/gencode.v22.annotation.chr20.gtf'", + "--direct_rna --minimap2_index_opts '-k 15' --sample_sheet 'differential_expression/sample_sheet.csv'", + "--jaffal_refBase 'differential_expression/chr20/' --jaffal_genome 'hg38_chr20' --jaffal_annotation 'genCode22'" ] agent = null container_sha = "shae7c9f184996a384e99be68e790f0612f0c732867" diff --git a/nextflow_schema.json b/nextflow_schema.json index 4bdabd7..df93b86 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -2,7 +2,8 @@ "$schema": "http://json-schema.org/draft-07/schema", "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", "title": "epi2me-labs/wf-transcriptomes", - "description": "Transcriptome analysis including gene fusions, differential expression as well as assembly and annotation of cDNA and direct RNA sequencing data.", + "workflow_title": "Workflow Transcriptomes", + "description": "Transcriptome analysis including assembly and annotation of cDNA and direct RNA sequencing data, gene fusions and differential expression.", "demo_url": "https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/wf-transcriptomes-demo.tar.gz", "aws_demo_url": "https://ont-exd-int-s3-euwst1-epi2me-labs.s3.amazonaws.com/wf-transcriptomes/wf-transcriptomes-demo/aws.nextflow.config", "url": "https://github.com/epi2me-labs/wf-transcriptomes", @@ -52,7 +53,7 @@ "title": "Reference annotation", "format": "file-path", "demo_data": "${projectDir}/test_data/SIRV_isoforms.gtf", - "description": "A reference annotation in GFF2 or GFF3 format (extensions .gtf(.gz), .gff(.gz), .gff3(.gz))", + "description": "A reference annotation in GFF2 or GFF3 format (extensions .gtf(.gz), .gff(.gz), .gff3(.gz)). Only annotation files from [Encode](https://www.encodeproject.org), [Ensembl](https://www.ensembl.org/index.html) and [NCBI](https://www.ncbi.nlm.nih.gov/) are supported.", "help_text": "This will be used for guiding the transcriptome assembly and to label transcripts with their corresponding gene identifiers." }, "direct_rna": { @@ -70,10 +71,13 @@ "help_text": "If selected and if the input is a multiplex directory the workflow will also process the unclassified directory." } }, - "required": [ - "fastq", - "transcriptome_source" - ] + "allOf": [ + { + "required": [ + "fastq", + "transcriptome_source" + ] + }] }, "output_options": { "title": "Output Options", @@ -354,8 +358,16 @@ "type": "boolean" } }, - "docs": { - "intro": "## Introduction\n\nThis workflow identifies RNA isoforms using either cDNA or direct RNA (dRNA) \nOxford Nanopore reads.\n\n### Preprocesing\ncDNA reads are initially preprocessed by [pychopper](https://github.com/epi2me-labs/pychopper) \nfor the identification of full-length reads, as well as trimming and orientation correction (This step is omitted for \n direct RNA reads).\n\n\n### Transcript assembly\n\n#### Reference-aided transcript assembly approach\n* Full length reads are mapped to a supplied reference genome using [minimap2](https://github.com/lh3/minimap2)\n* Transcripts are assembled by [stringtie](http://ccb.jhu.edu/software/stringtie) \nin long read mode (with or without a guide reference annotation) to generate the GFF annotation.\n* The annotation generated by the pipeline is compared to the reference annotation. \nusing [gffcompare](http://ccb.jhu.edu/software/stringtie/gffcompare.shtml)\n\n### Fusion gene detection\nFusion gene detection is performed using [JAFFA](https://github.com/Oshlack/JAFFA), with the JAFFAL extension for use \nwith ONT long reads. \n\n### Differential expression analysis\n\nDifferential gene expression (DGE) and differential transcript usage (DTU) analyses aim to identify genes and/or transcripts that show statistically altered expression patterns in a studied biological system. The results of the differential analyses are presented in a quantitative format and therefore the degree of change (up or down regulation) between experimental conditions can be calculated for each gene identified.\n\nThese differential analyses work by taking a \u201csnapshot\u201d of mRNA abundance and calculating the relative levels of transcripts and isoforms. In this context, expression corresponds to the number of messenger RNAs (mRNA) measured from each gene isoform within the organism / tissue / culture being investigated. In order to determine expression levels across the whole genome, sequence data specifically targeting the mRNA molecules can be generated.\n\nOxford Nanopore Technologies provides a number of sequencing solutions to allow users to generate the required snapshot of gene expression. This can be achieved by both sequencing the mRNA directly, or via a complementary DNA (cDNA) proxy. In contrast to short read sequencing technologies, entire mRNA transcripts can be captured as single reads. The example data provided with this tutorial is from a study based on the PCR-cDNA kit. This is a robust choice for performing differential transcript usage studies. This kit is suitable for preparation of sequence libraries from low mRNA input quantities. The cDNA population is enriched through PCR with low bias; an important prerequisite for the subsequent statistical analysis.\n\n[Workflow-transcriptomes](https://github.com/epi2me-labs/wf-transcriptomes) includes a subworkflow for DGE and DTU. The first step involves using either a reference alignment or _de novo_ assembly approach to create a set of mRNA sequences per sample. These are merged into a non-redundant transcriptome using [stringtie merge](http://ccb.jhu.edu/software/stringtie). The reads are then aligned to the transcriptome using minimap2 in a splice-aware manner. [Salmon](https://github.com/COMBINE-lab/salmon) is used for transcript quantification, giving per transcript counts and then the following R packages are used for analysis.\n\n### Pre-filtering of quantitative data using DRIMSeq\nDRIMSeq (Nowicka and Robinson (2016)) is used to filter the transcript count data from the salmon analysis. The filter step will be used to select for genes and transcripts that satisfy rules for the number of samples in which a gene or transcript must be observed and minimum threshold levels for the number of observed reads. The parameters used for filtering are defined in the config.yaml file. The default parameters defined for this analysis include\n* min_samps_gene_expr = 3 - a transcript must be mapped to a gene in at least this minimum number of samples for the gene be included in the analysis\n*\tmin_samps_feature_expr = 1 - a transcript must be mapped to an isoform in at least this this minimum number of samples for the gene isoform to be included in the analysis\n*\tmin_gene_expr = 10 - the minimum number of total mapped sequence reads for a gene to be considered expressed\n*\tmin_feature_expr = 3 - the minimum number of total mapped sequence reads for a gene isoform to be considered\n\n### edgeR based differential expression analysis\n+A statistical analysis is first performed using edgeR (Robinson, McCarthy, and Smyth (2010), McCarthy et al. (2012)) to identify the subset of differentially expressed genes. The filtered list of gene counts is used as input. A normalisation factor is calculated for each sequence library (using the default TMM method - please see McCarthy et al. (2012) for further details). The defined experimental design is used to calculate estimates of dispersion for each of the gene features. Statistical tests are calculated using the contrasts defined in the experimental design. The differentially expressed genes are corrected for false discovery (fdr) using the method of Benjamini & Hochberg (Benjamini and Hochberg (1995))\n\n### Differential transcript usage using DEXSeq\nDifferential transcript usage analysis is performed using the R DEXSeq package (Reyes et al. (2013)). Similar to the edgeR package, DEXSeq estimates the variance between the biological replicates and applies generalised linear models for the statistical testing. The key difference is that the DEXSeq method looks for differences at the exon count level. DEXSeq uses the filtered transcript count data prepared earlier in this analysis. \n\n### StageR stage-wise analysis of DGE and DTU\nThe final component of this isoform analysis is a stage-wise statistical test using the R software package `stageR` (Van den Berge and Clement (2018)). stageR uses (1) the raw p-values for DTU from the DEXSeq analysis in the previous section and (2) a false-discovery corrected set of p-values from testing whether individual genes contain at least one exon showing DTU. A hierarchical two-stage statistical testing evaluates the set of genes for DTU.\n\n## Running the workflow\nFor the differential expression analysis section you should have at least 3 repeats for each sample. \nYour FASTQ data will need to be organised in to 6 directories that represent 3 repeats for each condition. \n\n\n## Analysis \nDifferential gene expression is sensitive to the input data quantity and quality. There should be equivalence between samples in the number of sequence reads, mapped reads and quality scores. The sequence and alignment summary plots in the report can be used to assess these metrics. There is also a table that shows the transcript per million(TPM) calculated from the salmon counts. TPM normalizes the data for gene length and then sequencing depth, and makes it easier to compare across samples compared to counts.\n\n### Workflow inputs\n- Directory containing cDNA/direct RNA reads. Or a directory containing subdirectories each with reads from different samples\n (in fastq/fastq.gz format)\n- Reference genome in fasta format (required for reference-based assembly).\n- Optional reference annotation in GFF2/3 format (extensions allowed are .gtf(.gz), .gff(.gz), .gff3(.gz)) (required for differential expression analysis `--de_analysis`). Only annotation files from [Encode](https://www.encodeproject.org), [Ensembl](https://www.ensembl.org/index.html) and [NCBI](https://www.ncbi.nlm.nih.gov/) are supported.\n- For fusion detection, JAFFAL reference files (see Quickstart) \n", - "links": "## Useful links\n\n* [nextflow](https://www.nextflow.io/)\n* [docker](https://www.docker.com/products/docker-desktop)\n* [Singularity](https://sylabs.io/singularity/)\n* [racon](https://github.com/isovic/racon)" + "resources": { + "recommended": { + "cpus": 16, + "memory": "32GB" + }, + "minimum": { + "cpus": 8, + "memory": "16GB" + }, + "run_time": "15 minutes per sample, with 1 million reads and recommended resources.", + "arm_support": false } } \ No newline at end of file diff --git a/output_definition.json b/output_definition.json new file mode 100644 index 0000000..4d4955a --- /dev/null +++ b/output_definition.json @@ -0,0 +1,197 @@ +{ + "files": { + "workflow-report": { + "filepath": "wf-transcriptomes-report.html", + "title": "workflow report", + "description": "a HTML report document detailing the primary findings of the workflow", + "mime-type": "text/html", + "optional": false, + "type": "aggregated" + }, + "read-stats-per-file": { + "filepath": "fastq_ingress_results/reads/fastcat_stats/per-file-stats.tsv", + "title": "Per file read stats", + "description": "A TSV with per file read stats, including all samples.", + "mime-type": "text/tab-separated-values", + "optional": false, + "type": "aggregated" + }, + "read-stats-per-read": { + "filepath": "fastq_ingress_results/reads/fastcat_stats/per-read-stats.tsv", + "title": "Per file read stats", + "description": "A TSV with per read stats, including all samples.", + "mime-type": "text/tab-separated-values", + "optional": false, + "type": "aggregated" + }, + "run-ids": { + "filepath": "fastq_ingress_results/reads/fastcat_stats/run_ids", + "title": "Run ID's", + "description": "List of run IDs present in reads.", + "mime-type": "text/txt", + "optional": false, + "type": "aggregated" + }, + "metamap": { + "filepath": "fastq_ingress_results/reads/metamap.json", + "title": "Meta map json", + "description": "Metadata used in workflow presented in a JSON.", + "mime-type": "text/json", + "optional": false, + "type": "aggregated" + }, + "sample-data": { + "filepath": "fastq_ingress_results/reads/{{ alias }}.fastq.gz", + "title": "Concatenated sequence data", + "description": "Per sample reads concatenated in to one FASTQ file.", + "mime-type": "text/json", + "optional": false, + "type": "per-sample" + }, + "transcriptome": { + "filepath": "{{ alias }}_transcriptome.fas", + "title": "Assembled transcriptome", + "description": "Per sample assembled transcriptome.", + "mime-type": "text/x-fasta", + "optional": true, + "type": "per-sample" + }, + "merged_transcriptome": { + "filepath": "{{ alias }}_merged_transcriptome.fas", + "title": "Annotated assembled transcriptome", + "description": "Per sample annotated assembled transcriptome.", + "mime-type": "text/x-fasta", + "optional": true, + "type": "per-sample" + }, + "alignment-stats": { + "filepath": "{{ alias }}_read_aln_stats.tsv", + "title": "Alignment summary statistics", + "description": "Per sample alignment summary statistics.", + "mime-type": "text/tab-separated-valuesa", + "optional": false, + "type": "per-sample" + }, + "gff_compare": { + "filepath": "{{ alias }}_gffcompare", + "title": "GFF compare results.", + "description": "All GFF compare output files.", + "mime-type": "text/directory", + "optional": true, + "type": "per-sample" + }, + "dge-results-tsv": { + "filepath": "/de_analysis/results_dge.tsv", + "title": "Differential gene expression results", + "description": "This is a gene-level result file that describes genes and the probability that they show differential expression between experimental conditions .", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "dge-report-pdf": { + "filepath": "/de_analysis/results_dge.pdf", + "title": "Differential gene expression report", + "description": "Summary report of differential gene expression analysis as a PDF.", + "mime-type": "application/pdf", + "optional": true, + "type": "aggregated" + }, + "dtu-gene-tsv": { + "filepath": "/de_analysis/results_dtu_gene.tsv", + "title": "Differential transcript usage gene TSV", + "description": "This is a gene-level result file from DEXSeq that lists annotated genes and their probabilities of differential expression.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "dtu-report-pdf": { + "filepath": "/de_analysis/results_dtu.pdf", + "title": "Differential gene expression report", + "description": "Summary report of differential transcript usage results as a PDF.", + "mime-type": "application/pdf", + "optional": true, + "type": "aggregated" + }, + "dtu-transcript": { + "filepath": "/de_analysis/results_dtu_transcript.tsv", + "title": "Differential transcript usage TSV", + "description": "This is a transcript-level result file from DEXSeq that lists annotated genes and their probabilities of differential expression.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "dtu-stageR": { + "filepath": "/de_analysis/results_dtu_stageR.tsv ", + "title": "Differential transcript usage stageR TSV", + "description": "This is the output from StageR and it shows both gene and transcript probabilities of differential expression", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "dexseq": { + "filepath": "/de_analysis/results_dexseq.tsv", + "title": "Differential transcript usage DEXSeq TSV", + "description": "The complete output from the DEXSeq-analysis, shows both gene and transcript probabilities of differential expression.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "gene_counts": { + "filepath": "/de_analysis/all_gene_counts.tsv", + "title": "Gene counts", + "description": "Raw gene counts created by the Salmon tool, before filtering.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "transcipt_counts": { + "filepath": "/de_analysis/all_transcript_counts.tsv", + "title": "Transcript counts", + "description": "Raw transcript counts created by the Salmon tool, before filtering.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "transcipt_counts_filtered": { + "filepath": "/de_analysis/all_counts_filtered.tsv", + "title": "Transcript counts filtered", + "description": "Filtered transcript counts, used for DE_analysis.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "tpm_transcript_counts": { + "filepath": "/de_analysis/de_tpm_transcript_counts.tsv", + "title": "Transcript per million counts", + "description": "This file shows transcript per million (TPM) of the raw counts to facilitate comparisons across sample.", + "mime-type": "text/tab-separated-values", + "optional": true, + "type": "aggregated" + }, + "final_non_redundant_transcriptome": { + "filepath": "/de_analysis/final_non_redundant_transcriptome.fasta", + "title": "Final non redundant transcriptome", + "description": "Transcripts that were used for differential expression analysis including novel transcripts with the identifiers used for DE analysis.", + "mime-type": "text/x-fasta", + "optional": true, + "type": "aggregated" + }, + "jaffa_fasta": { + "filepath": "/jaffal_output_{{ alias }}/jaffa_results.fasta", + "title": "Fusion transcript sequences", + "description": "Fusion transcript sequences output by Jaffa.", + "mime-type": "text/x-fasta", + "optional": true, + "type": "per-sample" + }, + "jaffa_results": { + "filepath": "/jaffal_output_{{ alias }}/jaffa_results.csv", + "title": "Fusion transcript sequence summary file", + "description": "Fusion transcript sequences summary file output by Jaffa.", + "mime-type": "text/csv", + "optional": true, + "type": "per-sample" + } + + } +} \ No newline at end of file diff --git a/subworkflows/JAFFAL/gene_fusions.nf b/subworkflows/JAFFAL/gene_fusions.nf index e50f87d..e2b5013 100644 --- a/subworkflows/JAFFAL/gene_fusions.nf +++ b/subworkflows/JAFFAL/gene_fusions.nf @@ -1,6 +1,8 @@ process jaffal{ label "isoforms" + cpus 4 + memory "2 GB" input: tuple val(sample_id), path(fastq) path refBase diff --git a/subworkflows/differential_expression.nf b/subworkflows/differential_expression.nf index c6d0664..926780b 100644 --- a/subworkflows/differential_expression.nf +++ b/subworkflows/differential_expression.nf @@ -1,5 +1,7 @@ process checkSampleSheetCondition { label "isoforms" + cpus 1 + memory "2 GB" input: path "sample_sheet.csv" """ @@ -14,6 +16,7 @@ process count_transcripts { // library type is specified as forward stranded (-l SF) as it should have either been through pychopper or come from direct RNA reads. label "isoforms" cpus params.threads + memory "16 GB" input: tuple val(meta), path(bam), path(ref_transcriptome) output: @@ -29,6 +32,8 @@ process count_transcripts { process mergeCounts { label "isoforms" + cpus 1 + memory "2 GB" input: path counts output: @@ -40,6 +45,8 @@ process mergeCounts { process mergeTPM { label "isoforms" + cpus 1 + memory "2 GB" input: path counts output: @@ -54,6 +61,8 @@ process deAnalysis { label "isoforms" errorStrategy "retry" maxRetries 3 + cpus 4 + memory "16 GB" input: path sample_sheet path merged_tsv @@ -96,6 +105,8 @@ process deAnalysis { process plotResults { label "isoforms" + cpus 2 + memory "2 GB" input: path flt_count path res_dtu @@ -120,6 +131,7 @@ process build_minimap_index_transcriptome{ */ label "isoforms" cpus params.threads + memory "16 GB" input: path reference output: @@ -140,6 +152,7 @@ process map_transcriptome{ */ label "isoforms" cpus params.threads + memory "16 GB" input: tuple val(meta), path (fastq_reads), path(index) diff --git a/subworkflows/reference_assembly.nf b/subworkflows/reference_assembly.nf index 215af54..22fc83d 100644 --- a/subworkflows/reference_assembly.nf +++ b/subworkflows/reference_assembly.nf @@ -6,6 +6,7 @@ process map_reads{ */ label "isoforms" cpus params.threads + memory "16 GB" input: tuple val(sample_id), path (fastq_reads), path(index), path(reference)