Merge branch 'CW-2313' into 'dev'

CW-2313 remove condition sheet requirement

Closes CW-2313

See merge request epi2melabs/workflows/wf-transcriptomes!106

.gitlab-ci.yml

@@ -77,7 +77,7 @@ docker-run:
                   --de_analysis \
                   --ref_genome differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \
                 --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \
-                --direct_rna --minimap_index_opts '-k 15'"
+                --direct_rna --minimap_index_opts '-k 15'  --sample_sheet test_data/sample_sheet.csv"
               NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref
         - if: $MATRIX_NAME == "only_differential_expression"
           variables:
@@ -89,7 +89,7 @@ docker-run:
                 --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff \
                 --direct_rna --minimap_index_opts '-k 15' \
                 --ref_transcriptome differential_expression/ref_transcriptome.fasta \
-                --transcriptome_assembly false"
+                --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv"
               NF_IGNORE_PROCESSES: >
                 preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam
@@ -103,7 +103,7 @@ docker-run:
                 --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff3 \
                 --direct_rna --minimap_index_opts '-k 15' \
                 --ref_transcriptome differential_expression/ref_transcriptome.fasta \
-                --transcriptome_assembly false"
+                --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv"
               NF_IGNORE_PROCESSES: >
                 preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam
@@ -117,7 +117,7 @@ docker-run:
                   --ref_genome differential_expression/GRCh38.p14.NCBI_test.fna.gz \
                 --ref_annotation differential_expression/GRCh38.p14_NCBI_test.gtf.gz \
                 --direct_rna --minimap_index_opts '-w 25' \
-               --transcriptome_assembly false"
+               --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv"
               NF_IGNORE_PROCESSES: >
                 preprocess_reads,merge_transcriptomes,assemble_transcripts,
                 build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam           

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [unreleased]
 ### Changed
 - GitHub issue templates
+- Condition sheet is no longer required. The sample sheet is now used to indicate condition instead.
+    - For differential expression, the sample sheet must have a `condition` column to indicate which condition group each sample in the sample sheet belongs to.
+    - Values for the condition may be any two distinct strings, for example: treated/untreated; sample/control etc.
 
 ### Fixed
 - Remove default of null for `--ref_transcriptome`

README.md

@@ -207,26 +207,27 @@ __Note__: JAFFAL is not currently working on Mac M1 (osx-arm64 architecture).
 
 ### Differential Expression
 
-Differential Expression requires at least 2 replicates of each sample to compare. You can see an example condition_sheet.tsv in test_data.  
+Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below.
 
 **Example workflow for differential expression transcript assembly**
 
-#### Condition sheet
-The condition sheet should be a .tsv with two columns.
-- The sample_id column will need to match the 6 directories in the input fastq directory, if you are additionally using a sample_sheet they will need to correspond to the sample_ids in that.
-- The condition column will need to contain one of two keys to indicate the two samples being compared.
+#### Sample sheet condition column
+The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`.
+- Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory).
+- The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files.
+- The condition column will need to contain one of two keys to indicate the two samples being compared. for example: treated/untreated, sample/control etc.
 
-In the default `condition_sheet.tsv` available in the test_data directory we have used the following.
+In the default `sample_sheet.csv` available in the test_data directory we have used the following.
 
-eg. condition_sheet.tsv
+eg. sample_sheet.csv
 ```
-sample_id,condition
-barcode01,untreated
-barcode02,untreated
-barcode03,untreated
-barcode04,treated
-barcode05,treated
-barcode06,treated
+barcode,alias,condition
+barcode01,sample01,untreated
+barcode02,sample02,untreated
+barcode03,sample03,untreated
+barcode04,sample04,treated
+barcode05,sample05,treated
+barcode06,sample06,treated
 ```
 
 You will also need to provide a reference genome and a reference annotation file.

bin/de_analysis.R

@@ -15,7 +15,7 @@ cts <- as.matrix(read.csv("merged/all_counts.tsv", sep="\t", row.names="Referenc
 
 # Set up sample data frame:
 #changed this to sample_id
-coldata <- read.csv("de_analysis/coldata.tsv", row.names="sample_id", sep=",", stringsAsFactors=TRUE)
+coldata <- read.csv("de_analysis/coldata.tsv", row.names="alias", sep=",", stringsAsFactors=TRUE)
 
 coldata$sample_id <- rownames(coldata)
 coldata$condition <- factor(coldata$condition, levels=rev(levels(coldata$condition)))

bin/plot_dtu_results.R

@@ -5,7 +5,7 @@ suppressMessages(library(ggplot2))
 suppressMessages(library(tidyr))
 
 # Set up sample data frame:
-coldata <- read.csv("de_analysis/coldata.tsv", row.names="sample_id", sep=",")
+coldata <- read.csv("de_analysis/coldata.tsv", row.names="alias", sep=",")
 coldata$sample_id <- rownames(coldata)
 coldata$condition <- factor(coldata$condition, levels=rev(levels(coldata$condition)))
 coldata$type <-NULL

bin/workflow_glue/check_sample_sheet_condition.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""Check if a sample sheet is valid."""
+import csv
+import sys
+
+from .util import get_named_logger, wf_parser  # noqa: ABS101
+
+
+def main(args):
+    """Run the entry point."""
+    logger = get_named_logger("checkSheetCondition")
+    with open(args.sample_sheet, "r") as f:
+        csv_reader = csv.DictReader(f)
+        unique_controls = []
+        controls_dic = {}
+        for row in csv_reader:
+            if 'condition' in list(row.keys()):
+                unique_controls.append(row['condition'])
+                if row['condition'] not in controls_dic:
+                    controls_dic[row['condition']] = 1
+                else:
+                    controls_dic[row['condition']] += 1
+            else:
+                sys.exit(
+                    "Sample sheet has no condition column "
+                    "which is required for the "
+                    "differential expression subworkflow.")
+        if len(list(set(controls_dic.keys()))) != 2:
+            sys.exit(
+                "There must be only two unique conditions "
+                "in the condition column of the sample sheet.")
+        for val in list(controls_dic.values()):
+            if val < 2:
+                sys.exit(
+                    "There must be at least 2 repeats for each "
+                    "condition indicated in the sample sheet.")
+
+    logger.info(f"Checked sample sheet for condition column {args.sample_sheet}.")
+
+
+def argparser():
+    """Argument parser for entrypoint."""
+    parser = wf_parser("check_sample_sheet_condition")
+    parser.add_argument("sample_sheet", help="Sample sheet to check")
+    return parser

bin/workflow_glue/tests/conftest.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+"""Pytests argument definitions."""
+
+
+def pytest_addoption(parser):
+    """Define command line arguments for pytest."""
+    parser.addoption(
+        "--test_data",
+        action="store",
+        default="/host/test_data"
+    )

bin/workflow_glue/tests/test_check_sample_sheet_condition.py

@@ -0,0 +1,37 @@
+"""Test check_sample_sheet.py."""
+import os
+
+import pytest
+from workflow_glue import check_sample_sheet_condition
+
+
+# define a list of error messages
+ERROR_MESSAGES = [
+    ("sample_sheet_1.csv", "There must be only two unique conditions in the condition column of the sample sheet."),  # noqa: E501
+    ("sample_sheet_2.csv", "Sample sheet has no condition column which is required for the differential expression subworkflow."),  # noqa: E501
+    ("sample_sheet_3.csv", "There must be at least 2 repeats for each condition indicated in the sample sheet."),  # noqa: E501
+]
+
+
+@pytest.fixture
+def test_data(request):
+    """Define data location fixture."""
+    return os.path.join(
+        request.config.getoption("--test_data"),
+        "workflow_glue",
+        "check_sample_sheet_condition")
+
+
+@pytest.mark.parametrize("sample_sheet_name,error_msg", ERROR_MESSAGES)
+def test_check_sample_sheet(
+        test_data, sample_sheet_name, error_msg):
+    """Test the sample sheets."""
+    expected_error_message = error_msg
+    sample_sheet_path = f"{test_data}/{sample_sheet_name}"
+    args = check_sample_sheet_condition.argparser().parse_args(
+        [sample_sheet_path]
+    )
+    try:
+        check_sample_sheet_condition.main(args)
+    except SystemExit as e:
+        assert str(e) == expected_error_message

docs/quickstart.md

@@ -117,26 +117,27 @@ __Note__: JAFFAL is not currently working on Mac M1 (osx-arm64 architecture).
 
 ### Differential Expression
 
-Differential Expression requires at least 2 replicates of each sample to compare. You can see an example condition_sheet.tsv in test_data.  
+Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below.
 
 **Example workflow for differential expression transcript assembly**
 
-#### Condition sheet
-The condition sheet should be a .tsv with two columns.
-- The sample_id column will need to match the 6 directories in the input fastq directory, if you are additionally using a sample_sheet they will need to correspond to the sample_ids in that.
-- The condition column will need to contain one of two keys to indicate the two samples being compared.
+#### Sample sheet condition column
+The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`.
+- Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory).
+- The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files.
+- The condition column will need to contain one of two keys to indicate the two samples being compared. for example: treated/untreated, sample/control etc.
 
-In the default `condition_sheet.tsv` available in the test_data directory we have used the following.
+In the default `sample_sheet.csv` available in the test_data directory we have used the following.
 
-eg. condition_sheet.tsv
+eg. sample_sheet.csv
 ```
-sample_id,condition
-barcode01,untreated
-barcode02,untreated
-barcode03,untreated
-barcode04,treated
-barcode05,treated
-barcode06,treated
+barcode,alias,condition
+barcode01,sample01,untreated
+barcode02,sample02,untreated
+barcode03,sample03,untreated
+barcode04,sample04,treated
+barcode05,sample05,treated
+barcode06,sample06,treated
 ```
 
 You will also need to provide a reference genome and a reference annotation file.

main.nf

@@ -450,7 +450,6 @@ workflow pipeline {
         jaffal_refBase
         jaffal_genome
         jaffal_annotation
-        condition_sheet
         ref_transcriptome
         use_ref_ann
     main:
@@ -571,7 +570,7 @@ workflow pipeline {
 
 
         if (params.de_analysis){
-
+            sample_sheet = file(params.sample_sheet, type:"file")
             if (!params.ref_transcriptome){
                 merge_transcriptomes(run_gffcompare.output.gtf.collect(), ref_annotation, ref_genome)
                 transcriptome = merge_transcriptomes.out.fasta
@@ -581,13 +580,7 @@ workflow pipeline {
                 transcriptome = ref_transcriptome
                 gtf = ref_annotation
             }
-            check_match = Channel.fromPath(params.condition_sheet)
-            check_condition_sheet = check_match.splitCsv(header: true).map{ row -> tuple(
-                row.sample_id)
-            }
-            join_reads = input_reads.map{ meta, reads -> [meta.alias, reads]}
-            check_condition_sheet.join(join_reads, failOnMismatch: true)
-            de = differential_expression(transcriptome, input_reads, condition_sheet, gtf)
+            de = differential_expression(transcriptome, input_reads, sample_sheet, gtf)
             de_report = de.all_de
             count_transcripts_file = de.count_transcripts
             dtu_plots = de.dtu_plots
@@ -724,12 +717,12 @@ workflow {
         if (!params.ref_annotation){
             error = "You must provide a reference annotation."
         }
-        if (!params.condition_sheet){
-            error = "You must provide a condition_sheet or set de_analysis to false."
+        if (!params.sample_sheet){
+            error = "You must provide a sample_sheet with at least alias and condition columns."
+        }
+        if (params.containsKey("condition_sheet")) {
+        error = "Condition sheets have been deprecated. Please add a 'condition' column to your sample sheet instead. Check the quickstart for more information."
         }
-        condition_sheet = file(params.condition_sheet, type:"file")
-    } else{
-        condition_sheet = file("$projectDir/data/OPTIONAL_FILE")
     }
     if (error){
         throw new Exception(error)
@@ -744,7 +737,7 @@ workflow {
 
         pipeline(reads, ref_genome, ref_annotation,
             jaffal_refBase, params.jaffal_genome, params.jaffal_annotation,
-            condition_sheet, ref_transcriptome, use_ref_ann)
+            ref_transcriptome, use_ref_ann)
 
         output(pipeline.out.results)
     }

nextflow.config

@@ -87,7 +87,6 @@ params {
 
     // de options
     de_analysis = false
-    condition_sheet = "test_data/condition_sheet.tsv"
     ref_transcriptome = null
     min_samps_gene_expr	= 3
     min_samps_feature_expr = 1 

nextflow_schema.json

@@ -90,7 +90,7 @@
                 "sample_sheet": {
                     "type": "string",
                     "format": "file-path",
-                    "description": "A CSV file used to map barcodes to sample aliases. The sample sheet can be provided when the input data is a directory containing sub-directories with FASTQ files.",
+                    "description": "A CSV file used to map barcodes to sample aliases. The sample sheet can be provided when the input data is a directory containing sub-directories with FASTQ files. If you are running the differential expression workflow, there should be an additional column `condition` with any two distinct labels eg. `treated`,`untreated`. There should be at least 3 repeats for each condition.",
                     "help_text": "The sample sheet is a CSV file with, minimally, columns named `barcode` and `alias`. Extra columns are allowed. A `type` column is required for certain workflows and should have the following values; `test_sample`, `positive_control`, `negative_control`, `no_template_control`."
                 },
                 "sample": {
@@ -217,13 +217,6 @@
                     "description": "Run DE anaylsis",
                     "help_text": "Running this requires you to provide at least two replicates for a control and treated sample as well as a condition sheet param."
                 },
-                "condition_sheet": {
-                    "type": "string",
-                    "format": "file-path",
-                    "description": "CSV file with sample_id, condition",
-                    "default": "null",
-                    "help_text": "The condition sheet should be a headed CSV file with two columns sample_id,condition. Should be at least 3 repeats for each condition."
-                },
                 "min_gene_expr": {
                     "type": "integer",
                     "default": 10,

subworkflows/differential_expression.nf

@@ -1,3 +1,14 @@
+process checkSampleSheetCondition {
+    label "isoforms"
+    input:
+        path "sample_sheet.csv"
+    """
+    workflow-glue check_sample_sheet_condition "sample_sheet.csv"
+    """
+}
+
+
+
 process count_transcripts {
     // Count transcripts using Salmon.
     // library type is specified as forward stranded (-l SF) as it should have either been through pychopper or come from direct RNA reads.
@@ -44,7 +55,7 @@ process deAnalysis {
     errorStrategy "retry"
     maxRetries 1
     input:
-        path condition_sheet
+        path sample_sheet
         path merged_tsv 
         path "annotation.gtf"
     output:
@@ -64,7 +75,7 @@ process deAnalysis {
     mkdir merged
     mkdir de_analysis
     mv $merged_tsv merged/all_counts.tsv
-    mv $condition_sheet de_analysis/coldata.tsv
+    mv $sample_sheet de_analysis/coldata.tsv
     de_analysis.R annotation.gtf $params.min_samps_gene_expr $params.min_samps_feature_expr $params.min_gene_expr $params.min_feature_expr $annotation_type
    
     """
@@ -76,18 +87,18 @@ process plotResults {
     input:
         path flt_count
         path res_dtu
-        path condition_sheet 
+        path sample_sheet 
         path de_analysis
     output:
         path "de_analysis/dtu_plots.pdf", emit: dtu_plots
-        path "condition_sheet.tsv", emit: condition_sheet_tsv
+        path "sample_sheet.tsv", emit: sample_sheet_csv
         path "de_analysis", emit: stageR
     """
     mkdir merged
-    mv $condition_sheet de_analysis/coldata.tsv
+    mv $sample_sheet de_analysis/coldata.tsv
     mv $flt_count merged/all_counts_filtered.tsv
     plot_dtu_results.R
-    mv de_analysis/coldata.tsv condition_sheet.tsv
+    mv de_analysis/coldata.tsv sample_sheet.tsv
     """
 }
 
@@ -134,18 +145,19 @@ workflow differential_expression {
     take:
        ref_transcriptome
        full_len_reads
-       condition_sheet
+       sample_sheet
        ref_annotation
     main:
+        checkSampleSheetCondition(sample_sheet)
         t_index = build_minimap_index_transcriptome(ref_transcriptome)
         mapped = map_transcriptome(full_len_reads.combine(t_index))
         count_transcripts(mapped.bam.combine(t_index.map{ mmi, reference -> reference}))
         merged = mergeCounts(count_transcripts.out.counts.collect())
         merged_TPM = mergeTPM(count_transcripts.out.counts.collect())
-        analysis = deAnalysis(condition_sheet, merged, ref_annotation)
-        plotResults(analysis.flt_counts, analysis.stageR, condition_sheet, analysis.de_analysis)
+        analysis = deAnalysis(sample_sheet, merged, ref_annotation)
+        plotResults(analysis.flt_counts, analysis.stageR, sample_sheet, analysis.de_analysis)
         de_report = analysis.flt_counts.combine(analysis.gene_counts).combine(analysis.dge).combine(analysis.dexseq).combine(
-                    analysis.stageR).combine(plotResults.out.condition_sheet_tsv).combine(merged).combine(
+                    analysis.stageR).combine(plotResults.out.sample_sheet_csv).combine(merged).combine(
                     ref_annotation).combine(merged_TPM)
         count_transcripts_file = count_transcripts.out.seqkit_stats.collect()
 emit:

test_data/condition_sheet.tsv

@@ -1,7 +0,0 @@
-sample_id,condition
-barcode01,untreated
-barcode02,untreated
-barcode03,untreated
-barcode04,treated
-barcode05,treated
-barcode06,treated

test_data/sample_sheet

@@ -1,3 +0,0 @@
-barcode,sample_id,alias,type
-barcode01,SRR12480552,SRR12480552,test_sample1
-barcode02,SRR12447502,SRR12447502,test_sample2

test_data/sample_sheet.csv

@@ -0,0 +1,7 @@
+barcode,sample_id,alias,condition
+barcode01,sample01,sample01,untreated
+barcode02,sample02,sample02,untreated
+barcode03,sample03,sample03,untreated
+barcode04,sample04,sample04,treated
+barcode05,sample05,sample05,treated
+barcode06,sample06,sample06,treated

test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_1.csv

@@ -0,0 +1,7 @@
+barcode,sample_id,alias,condition
+barcode01,sample01,sample01,untreated
+barcode02,sample02,sample02,untreated
+barcode03,sample03,sample03,untreated
+barcode04,sample04,sample04,treated
+barcode05,sample05,sample05,treated
+barcode06,sample06,sample06,other

test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_2.csv

@@ -0,0 +1,7 @@
+barcode,sample_id,alias
+barcode01,sample01,sample01
+barcode02,sample02,sample02
+barcode03,sample03,sample03
+barcode04,sample04,sample04
+barcode05,sample05,sample05
+barcode06,sample06,sample06

test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_3.csv

@@ -0,0 +1,3 @@
+barcode,sample_id,alias,condition
+barcode01,sample01,sample01,untreated
+barcode04,sample04,sample04,treated