From df6c9a2a31b5c8be2316be6c12ed40030eb5fcc1 Mon Sep 17 00:00:00 2001 From: Sarah Griffiths Date: Mon, 10 Jul 2023 16:45:32 +0000 Subject: [PATCH] CW-2313 remove condition sheet requirement --- .gitlab-ci.yml | 8 ++-- CHANGELOG.md | 3 ++ README.md | 29 ++++++------ bin/de_analysis.R | 2 +- bin/plot_dtu_results.R | 2 +- .../check_sample_sheet_condition.py | 45 +++++++++++++++++++ bin/workflow_glue/tests/conftest.py | 11 +++++ .../test_check_sample_sheet_condition.py | 37 +++++++++++++++ docs/quickstart.md | 29 ++++++------ main.nf | 23 ++++------ nextflow.config | 1 - nextflow_schema.json | 9 +--- subworkflows/differential_expression.nf | 32 ++++++++----- test_data/condition_sheet.tsv | 7 --- test_data/sample_sheet | 3 -- test_data/sample_sheet.csv | 7 +++ .../sample_sheet_1.csv | 7 +++ .../sample_sheet_2.csv | 7 +++ .../sample_sheet_3.csv | 3 ++ 19 files changed, 187 insertions(+), 78 deletions(-) create mode 100755 bin/workflow_glue/check_sample_sheet_condition.py create mode 100755 bin/workflow_glue/tests/conftest.py create mode 100755 bin/workflow_glue/tests/test_check_sample_sheet_condition.py delete mode 100644 test_data/condition_sheet.tsv delete mode 100644 test_data/sample_sheet create mode 100644 test_data/sample_sheet.csv create mode 100644 test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_1.csv create mode 100644 test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_2.csv create mode 100644 test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_3.csv diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33bb5a3..4915790 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ docker-run: --de_analysis \ --ref_genome differential_expression/hg38_chr20.fa --transcriptome-source reference-guided \ --ref_annotation differential_expression/gencode.v22.annotation.chr20.gtf \ - --direct_rna --minimap_index_opts '-k 15'" + --direct_rna --minimap_index_opts '-k 15' --sample_sheet test_data/sample_sheet.csv" NF_IGNORE_PROCESSES: preprocess_reads,merge_transcriptomes,decompress_annotation,decompress_ref - if: $MATRIX_NAME == "only_differential_expression" variables: @@ -89,7 +89,7 @@ docker-run: --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff \ --direct_rna --minimap_index_opts '-k 15' \ --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --transcriptome_assembly false" + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam @@ -103,7 +103,7 @@ docker-run: --ref_annotation differential_expression/gencode.v22.annotation.chr20.gff3 \ --direct_rna --minimap_index_opts '-k 15' \ --ref_transcriptome differential_expression/ref_transcriptome.fasta \ - --transcriptome_assembly false" + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts,decompress_annotation,decompress_ref, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam @@ -117,7 +117,7 @@ docker-run: --ref_genome differential_expression/GRCh38.p14.NCBI_test.fna.gz \ --ref_annotation differential_expression/GRCh38.p14_NCBI_test.gtf.gz \ --direct_rna --minimap_index_opts '-w 25' \ - --transcriptome_assembly false" + --transcriptome_assembly false --sample_sheet test_data/sample_sheet.csv" NF_IGNORE_PROCESSES: > preprocess_reads,merge_transcriptomes,assemble_transcripts, build_minimap_index,get_transcriptome,merge_gff_bundles,run_gffcompare,build_minimap_index,split_bam diff --git a/CHANGELOG.md b/CHANGELOG.md index 9343e08..168121f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Changed - GitHub issue templates +- Condition sheet is no longer required. The sample sheet is now used to indicate condition instead. + - For differential expression, the sample sheet must have a `condition` column to indicate which condition group each sample in the sample sheet belongs to. + - Values for the condition may be any two distinct strings, for example: treated/untreated; sample/control etc. ### Fixed - Remove default of null for `--ref_transcriptome` diff --git a/README.md b/README.md index 6a59627..7ed3a34 100644 --- a/README.md +++ b/README.md @@ -207,26 +207,27 @@ __Note__: JAFFAL is not currently working on Mac M1 (osx-arm64 architecture). ### Differential Expression -Differential Expression requires at least 2 replicates of each sample to compare. You can see an example condition_sheet.tsv in test_data. +Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below. **Example workflow for differential expression transcript assembly** -#### Condition sheet -The condition sheet should be a .tsv with two columns. -- The sample_id column will need to match the 6 directories in the input fastq directory, if you are additionally using a sample_sheet they will need to correspond to the sample_ids in that. -- The condition column will need to contain one of two keys to indicate the two samples being compared. +#### Sample sheet condition column +The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`. +- Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory). +- The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files. +- The condition column will need to contain one of two keys to indicate the two samples being compared. for example: treated/untreated, sample/control etc. -In the default `condition_sheet.tsv` available in the test_data directory we have used the following. +In the default `sample_sheet.csv` available in the test_data directory we have used the following. -eg. condition_sheet.tsv +eg. sample_sheet.csv ``` -sample_id,condition -barcode01,untreated -barcode02,untreated -barcode03,untreated -barcode04,treated -barcode05,treated -barcode06,treated +barcode,alias,condition +barcode01,sample01,untreated +barcode02,sample02,untreated +barcode03,sample03,untreated +barcode04,sample04,treated +barcode05,sample05,treated +barcode06,sample06,treated ``` You will also need to provide a reference genome and a reference annotation file. diff --git a/bin/de_analysis.R b/bin/de_analysis.R index 83059b0..5dceb26 100755 --- a/bin/de_analysis.R +++ b/bin/de_analysis.R @@ -15,7 +15,7 @@ cts <- as.matrix(read.csv("merged/all_counts.tsv", sep="\t", row.names="Referenc # Set up sample data frame: #changed this to sample_id -coldata <- read.csv("de_analysis/coldata.tsv", row.names="sample_id", sep=",", stringsAsFactors=TRUE) +coldata <- read.csv("de_analysis/coldata.tsv", row.names="alias", sep=",", stringsAsFactors=TRUE) coldata$sample_id <- rownames(coldata) coldata$condition <- factor(coldata$condition, levels=rev(levels(coldata$condition))) diff --git a/bin/plot_dtu_results.R b/bin/plot_dtu_results.R index ec90582..4f171e6 100755 --- a/bin/plot_dtu_results.R +++ b/bin/plot_dtu_results.R @@ -5,7 +5,7 @@ suppressMessages(library(ggplot2)) suppressMessages(library(tidyr)) # Set up sample data frame: -coldata <- read.csv("de_analysis/coldata.tsv", row.names="sample_id", sep=",") +coldata <- read.csv("de_analysis/coldata.tsv", row.names="alias", sep=",") coldata$sample_id <- rownames(coldata) coldata$condition <- factor(coldata$condition, levels=rev(levels(coldata$condition))) coldata$type <-NULL diff --git a/bin/workflow_glue/check_sample_sheet_condition.py b/bin/workflow_glue/check_sample_sheet_condition.py new file mode 100755 index 0000000..b02617e --- /dev/null +++ b/bin/workflow_glue/check_sample_sheet_condition.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +"""Check if a sample sheet is valid.""" +import csv +import sys + +from .util import get_named_logger, wf_parser # noqa: ABS101 + + +def main(args): + """Run the entry point.""" + logger = get_named_logger("checkSheetCondition") + with open(args.sample_sheet, "r") as f: + csv_reader = csv.DictReader(f) + unique_controls = [] + controls_dic = {} + for row in csv_reader: + if 'condition' in list(row.keys()): + unique_controls.append(row['condition']) + if row['condition'] not in controls_dic: + controls_dic[row['condition']] = 1 + else: + controls_dic[row['condition']] += 1 + else: + sys.exit( + "Sample sheet has no condition column " + "which is required for the " + "differential expression subworkflow.") + if len(list(set(controls_dic.keys()))) != 2: + sys.exit( + "There must be only two unique conditions " + "in the condition column of the sample sheet.") + for val in list(controls_dic.values()): + if val < 2: + sys.exit( + "There must be at least 2 repeats for each " + "condition indicated in the sample sheet.") + + logger.info(f"Checked sample sheet for condition column {args.sample_sheet}.") + + +def argparser(): + """Argument parser for entrypoint.""" + parser = wf_parser("check_sample_sheet_condition") + parser.add_argument("sample_sheet", help="Sample sheet to check") + return parser diff --git a/bin/workflow_glue/tests/conftest.py b/bin/workflow_glue/tests/conftest.py new file mode 100755 index 0000000..f16dc78 --- /dev/null +++ b/bin/workflow_glue/tests/conftest.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +"""Pytests argument definitions.""" + + +def pytest_addoption(parser): + """Define command line arguments for pytest.""" + parser.addoption( + "--test_data", + action="store", + default="/host/test_data" + ) diff --git a/bin/workflow_glue/tests/test_check_sample_sheet_condition.py b/bin/workflow_glue/tests/test_check_sample_sheet_condition.py new file mode 100755 index 0000000..af414cc --- /dev/null +++ b/bin/workflow_glue/tests/test_check_sample_sheet_condition.py @@ -0,0 +1,37 @@ +"""Test check_sample_sheet.py.""" +import os + +import pytest +from workflow_glue import check_sample_sheet_condition + + +# define a list of error messages +ERROR_MESSAGES = [ + ("sample_sheet_1.csv", "There must be only two unique conditions in the condition column of the sample sheet."), # noqa: E501 + ("sample_sheet_2.csv", "Sample sheet has no condition column which is required for the differential expression subworkflow."), # noqa: E501 + ("sample_sheet_3.csv", "There must be at least 2 repeats for each condition indicated in the sample sheet."), # noqa: E501 +] + + +@pytest.fixture +def test_data(request): + """Define data location fixture.""" + return os.path.join( + request.config.getoption("--test_data"), + "workflow_glue", + "check_sample_sheet_condition") + + +@pytest.mark.parametrize("sample_sheet_name,error_msg", ERROR_MESSAGES) +def test_check_sample_sheet( + test_data, sample_sheet_name, error_msg): + """Test the sample sheets.""" + expected_error_message = error_msg + sample_sheet_path = f"{test_data}/{sample_sheet_name}" + args = check_sample_sheet_condition.argparser().parse_args( + [sample_sheet_path] + ) + try: + check_sample_sheet_condition.main(args) + except SystemExit as e: + assert str(e) == expected_error_message diff --git a/docs/quickstart.md b/docs/quickstart.md index 392ea2e..9ef5b36 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -117,26 +117,27 @@ __Note__: JAFFAL is not currently working on Mac M1 (osx-arm64 architecture). ### Differential Expression -Differential Expression requires at least 2 replicates of each sample to compare. You can see an example condition_sheet.tsv in test_data. +Differential Expression requires at least 2 replicates of each sample to compare (but we recommend three). You can see an example sample_sheet.csv below. **Example workflow for differential expression transcript assembly** -#### Condition sheet -The condition sheet should be a .tsv with two columns. -- The sample_id column will need to match the 6 directories in the input fastq directory, if you are additionally using a sample_sheet they will need to correspond to the sample_ids in that. -- The condition column will need to contain one of two keys to indicate the two samples being compared. +#### Sample sheet condition column +The sample sheet should be a comma separated values file (.csv) and include at least three columns named `barcode`, `alias` and `condition`. +- Each `barcode` should refer to a directory of the same name in the input FASTQ directory (in the example below `barcode01` to `barcode06` reflect the `test_data` directory). +- The `alias` column allows you to rename each barcode to an alias that will be used in the report and other output files. +- The condition column will need to contain one of two keys to indicate the two samples being compared. for example: treated/untreated, sample/control etc. -In the default `condition_sheet.tsv` available in the test_data directory we have used the following. +In the default `sample_sheet.csv` available in the test_data directory we have used the following. -eg. condition_sheet.tsv +eg. sample_sheet.csv ``` -sample_id,condition -barcode01,untreated -barcode02,untreated -barcode03,untreated -barcode04,treated -barcode05,treated -barcode06,treated +barcode,alias,condition +barcode01,sample01,untreated +barcode02,sample02,untreated +barcode03,sample03,untreated +barcode04,sample04,treated +barcode05,sample05,treated +barcode06,sample06,treated ``` You will also need to provide a reference genome and a reference annotation file. diff --git a/main.nf b/main.nf index 3cf0d15..103dd8a 100644 --- a/main.nf +++ b/main.nf @@ -450,7 +450,6 @@ workflow pipeline { jaffal_refBase jaffal_genome jaffal_annotation - condition_sheet ref_transcriptome use_ref_ann main: @@ -571,7 +570,7 @@ workflow pipeline { if (params.de_analysis){ - + sample_sheet = file(params.sample_sheet, type:"file") if (!params.ref_transcriptome){ merge_transcriptomes(run_gffcompare.output.gtf.collect(), ref_annotation, ref_genome) transcriptome = merge_transcriptomes.out.fasta @@ -581,13 +580,7 @@ workflow pipeline { transcriptome = ref_transcriptome gtf = ref_annotation } - check_match = Channel.fromPath(params.condition_sheet) - check_condition_sheet = check_match.splitCsv(header: true).map{ row -> tuple( - row.sample_id) - } - join_reads = input_reads.map{ meta, reads -> [meta.alias, reads]} - check_condition_sheet.join(join_reads, failOnMismatch: true) - de = differential_expression(transcriptome, input_reads, condition_sheet, gtf) + de = differential_expression(transcriptome, input_reads, sample_sheet, gtf) de_report = de.all_de count_transcripts_file = de.count_transcripts dtu_plots = de.dtu_plots @@ -724,12 +717,12 @@ workflow { if (!params.ref_annotation){ error = "You must provide a reference annotation." } - if (!params.condition_sheet){ - error = "You must provide a condition_sheet or set de_analysis to false." + if (!params.sample_sheet){ + error = "You must provide a sample_sheet with at least alias and condition columns." + } + if (params.containsKey("condition_sheet")) { + error = "Condition sheets have been deprecated. Please add a 'condition' column to your sample sheet instead. Check the quickstart for more information." } - condition_sheet = file(params.condition_sheet, type:"file") - } else{ - condition_sheet = file("$projectDir/data/OPTIONAL_FILE") } if (error){ throw new Exception(error) @@ -744,7 +737,7 @@ workflow { pipeline(reads, ref_genome, ref_annotation, jaffal_refBase, params.jaffal_genome, params.jaffal_annotation, - condition_sheet, ref_transcriptome, use_ref_ann) + ref_transcriptome, use_ref_ann) output(pipeline.out.results) } diff --git a/nextflow.config b/nextflow.config index e3d10aa..c903544 100644 --- a/nextflow.config +++ b/nextflow.config @@ -87,7 +87,6 @@ params { // de options de_analysis = false - condition_sheet = "test_data/condition_sheet.tsv" ref_transcriptome = null min_samps_gene_expr = 3 min_samps_feature_expr = 1 diff --git a/nextflow_schema.json b/nextflow_schema.json index 08fcf66..5654cf0 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -90,7 +90,7 @@ "sample_sheet": { "type": "string", "format": "file-path", - "description": "A CSV file used to map barcodes to sample aliases. The sample sheet can be provided when the input data is a directory containing sub-directories with FASTQ files.", + "description": "A CSV file used to map barcodes to sample aliases. The sample sheet can be provided when the input data is a directory containing sub-directories with FASTQ files. If you are running the differential expression workflow, there should be an additional column `condition` with any two distinct labels eg. `treated`,`untreated`. There should be at least 3 repeats for each condition.", "help_text": "The sample sheet is a CSV file with, minimally, columns named `barcode` and `alias`. Extra columns are allowed. A `type` column is required for certain workflows and should have the following values; `test_sample`, `positive_control`, `negative_control`, `no_template_control`." }, "sample": { @@ -217,13 +217,6 @@ "description": "Run DE anaylsis", "help_text": "Running this requires you to provide at least two replicates for a control and treated sample as well as a condition sheet param." }, - "condition_sheet": { - "type": "string", - "format": "file-path", - "description": "CSV file with sample_id, condition", - "default": "null", - "help_text": "The condition sheet should be a headed CSV file with two columns sample_id,condition. Should be at least 3 repeats for each condition." - }, "min_gene_expr": { "type": "integer", "default": 10, diff --git a/subworkflows/differential_expression.nf b/subworkflows/differential_expression.nf index f421a86..cb992c1 100644 --- a/subworkflows/differential_expression.nf +++ b/subworkflows/differential_expression.nf @@ -1,3 +1,14 @@ +process checkSampleSheetCondition { + label "isoforms" + input: + path "sample_sheet.csv" + """ + workflow-glue check_sample_sheet_condition "sample_sheet.csv" + """ +} + + + process count_transcripts { // Count transcripts using Salmon. // library type is specified as forward stranded (-l SF) as it should have either been through pychopper or come from direct RNA reads. @@ -44,7 +55,7 @@ process deAnalysis { errorStrategy "retry" maxRetries 1 input: - path condition_sheet + path sample_sheet path merged_tsv path "annotation.gtf" output: @@ -64,7 +75,7 @@ process deAnalysis { mkdir merged mkdir de_analysis mv $merged_tsv merged/all_counts.tsv - mv $condition_sheet de_analysis/coldata.tsv + mv $sample_sheet de_analysis/coldata.tsv de_analysis.R annotation.gtf $params.min_samps_gene_expr $params.min_samps_feature_expr $params.min_gene_expr $params.min_feature_expr $annotation_type """ @@ -76,18 +87,18 @@ process plotResults { input: path flt_count path res_dtu - path condition_sheet + path sample_sheet path de_analysis output: path "de_analysis/dtu_plots.pdf", emit: dtu_plots - path "condition_sheet.tsv", emit: condition_sheet_tsv + path "sample_sheet.tsv", emit: sample_sheet_csv path "de_analysis", emit: stageR """ mkdir merged - mv $condition_sheet de_analysis/coldata.tsv + mv $sample_sheet de_analysis/coldata.tsv mv $flt_count merged/all_counts_filtered.tsv plot_dtu_results.R - mv de_analysis/coldata.tsv condition_sheet.tsv + mv de_analysis/coldata.tsv sample_sheet.tsv """ } @@ -134,18 +145,19 @@ workflow differential_expression { take: ref_transcriptome full_len_reads - condition_sheet + sample_sheet ref_annotation main: + checkSampleSheetCondition(sample_sheet) t_index = build_minimap_index_transcriptome(ref_transcriptome) mapped = map_transcriptome(full_len_reads.combine(t_index)) count_transcripts(mapped.bam.combine(t_index.map{ mmi, reference -> reference})) merged = mergeCounts(count_transcripts.out.counts.collect()) merged_TPM = mergeTPM(count_transcripts.out.counts.collect()) - analysis = deAnalysis(condition_sheet, merged, ref_annotation) - plotResults(analysis.flt_counts, analysis.stageR, condition_sheet, analysis.de_analysis) + analysis = deAnalysis(sample_sheet, merged, ref_annotation) + plotResults(analysis.flt_counts, analysis.stageR, sample_sheet, analysis.de_analysis) de_report = analysis.flt_counts.combine(analysis.gene_counts).combine(analysis.dge).combine(analysis.dexseq).combine( - analysis.stageR).combine(plotResults.out.condition_sheet_tsv).combine(merged).combine( + analysis.stageR).combine(plotResults.out.sample_sheet_csv).combine(merged).combine( ref_annotation).combine(merged_TPM) count_transcripts_file = count_transcripts.out.seqkit_stats.collect() emit: diff --git a/test_data/condition_sheet.tsv b/test_data/condition_sheet.tsv deleted file mode 100644 index 0c35da6..0000000 --- a/test_data/condition_sheet.tsv +++ /dev/null @@ -1,7 +0,0 @@ -sample_id,condition -barcode01,untreated -barcode02,untreated -barcode03,untreated -barcode04,treated -barcode05,treated -barcode06,treated diff --git a/test_data/sample_sheet b/test_data/sample_sheet deleted file mode 100644 index 4cfbd82..0000000 --- a/test_data/sample_sheet +++ /dev/null @@ -1,3 +0,0 @@ -barcode,sample_id,alias,type -barcode01,SRR12480552,SRR12480552,test_sample1 -barcode02,SRR12447502,SRR12447502,test_sample2 diff --git a/test_data/sample_sheet.csv b/test_data/sample_sheet.csv new file mode 100644 index 0000000..00adea1 --- /dev/null +++ b/test_data/sample_sheet.csv @@ -0,0 +1,7 @@ +barcode,sample_id,alias,condition +barcode01,sample01,sample01,untreated +barcode02,sample02,sample02,untreated +barcode03,sample03,sample03,untreated +barcode04,sample04,sample04,treated +barcode05,sample05,sample05,treated +barcode06,sample06,sample06,treated \ No newline at end of file diff --git a/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_1.csv b/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_1.csv new file mode 100644 index 0000000..3a0f657 --- /dev/null +++ b/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_1.csv @@ -0,0 +1,7 @@ +barcode,sample_id,alias,condition +barcode01,sample01,sample01,untreated +barcode02,sample02,sample02,untreated +barcode03,sample03,sample03,untreated +barcode04,sample04,sample04,treated +barcode05,sample05,sample05,treated +barcode06,sample06,sample06,other \ No newline at end of file diff --git a/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_2.csv b/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_2.csv new file mode 100644 index 0000000..f7c55b7 --- /dev/null +++ b/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_2.csv @@ -0,0 +1,7 @@ +barcode,sample_id,alias +barcode01,sample01,sample01 +barcode02,sample02,sample02 +barcode03,sample03,sample03 +barcode04,sample04,sample04 +barcode05,sample05,sample05 +barcode06,sample06,sample06 \ No newline at end of file diff --git a/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_3.csv b/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_3.csv new file mode 100644 index 0000000..ecc8f39 --- /dev/null +++ b/test_data/workflow_glue/check_sample_sheet_condition/sample_sheet_3.csv @@ -0,0 +1,3 @@ +barcode,sample_id,alias,condition +barcode01,sample01,sample01,untreated +barcode04,sample04,sample04,treated