diff --git a/README.md b/README.md index ed24e63..31fc58b 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ The workflow can currently be run using either the required software. Both methods are automated out-of-the-box provided either docker of conda is installed. -> See the sections below for installation of these prerequisites in various scenarios. -> It is not required to clone or download the git repository in order to run the workflow. +It is not required to clone or download the git repository in order to run the workflow. +For more information on running EPI2ME Labs workflows [visit out website](https://labs.epi2me.io/wfindex). **Workflow options** @@ -39,192 +39,6 @@ The primary outputs of the workflow include: * an HTML report document detailing the primary findings of the workflow. -### Supported installations and GridION devices - -Installation of the software on a GridION can be performed using the command - -`sudo apt install ont-nextflow` - -This will install a java runtime, Nextflow and docker. If *docker* has not already been -configured the command below can be used to provide user access to the *docker* -services. Please logout of your computer after this command has been typed. - -`sudo usermod -aG docker $USER` - -### Installation on Ubuntu devices - -For hardware running Ubuntu the following instructions should suffice to install -Nextflow and Docker in order to run the workflow. - -1. Install a Jva runtime environment (JRE): - - ```sudo apt install default-jre``` - -2. Download and install Nextflow may be downloaded from https://www.nextflow.io: - - ```curl -s https://get.nextflow.io | bash``` - - This will place a `nextflow` binary in the current working directory, you - may wish to move this to a location where it is always accessible, e.g: - - ```sudo mv nextflow /usr/local/bin``` - -3. Install docker and add the current user to the docker group to enable access: - - ``` - sudo apt install docker.io - sudo usermod -aG docker $USER - ``` - -## Running the workflow - -The `wf-template` workflow can be controlled by the following parameters. The `fastq` parameter -is the most important parameter: it is required to identify the location of the -sequence files to be analysed. - -**Parameters:** - -- `fastq` specifies a *directory* path to FASTQ files (required) -- `out_dir` the path for the output (default: output) - -To run the workflow using Docker containers supply the `-profile standard` -argument to `nextflow run`: - -> The command below uses test data available from the [github repository](https://github.com/epi2me-labs/wf-template/tree/master/test_data) -> It can be obtained with `git clone https://github.com/epi2me-labs/wf-template`. - -``` -# run the pipeline with the test data -OUTPUT=output -nextflow run epi2me-labs/wf-template \ - -w ${OUTPUT}/workspace \ - -profile standard \ - --fastq test_data \ - --out_dir ${OUTPUT} -``` - -The output of the pipeline will be found in `./output` for the above -example. This directory contains the nextflow working directories alongside -the two primary outputs of the pipeline: a `seqs.txt` file containing a summary -of all reads, and a `report.html` file summarising the workflows calculations. - -### Running the workflow with Conda - -To run the workflow using conda rather than docker, simply replace - - -profile standard - -with - - -profile conda - -in the command above. - -### Configuration and tuning - -> This section provides some minimal guidance for changing common options, see -> the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for further details. - -The default settings for the workflow are described in the configuration file `nextflow.config` -found within the git repository. The default configuration defines an *executor* that will -use a specified maximum CPU cores (four at the time of writing) and RAM (eight gigabytes). - -If the workflow is being run on a device other than a GridION, the available memory and -number of CPUs may be adjusted to the available number of CPU cores. This can be done by -creating a file `my_config.cfg` in the working directory with the following contents: - -``` -executor { - $local { - cpus = 4 - memory = "8 GB" - } -} -``` - -and running the workflow providing the `-c` (config) option, e.g.: - -``` -# run the pipeline with custom configuration -nextflow run epi2me-labs/wf-template \ - -c my_config.cfg \ - ... -``` - -The contents of the `my_config.cfg` file will override the contents of the default -configuration file. See the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) -for more information concerning customized configuration. - -**Using a fixed conda environment** - -By default, Nextflow will attempt to create a fresh conda environment for any new -analysis (for reasons of reproducibility). This may be undesirable if many analyses -are being run. To avoid the situation a fixed conda environment can be used for all -analyses by creating a custom config with the following stanza: - -``` -profiles { - // profile using conda environments rather than docker - // containers - fixed_conda { - docker { - enabled = false - } - process { - withLabel:artic { - conda = "/path/to/my/conda/environment" - } - shell = ['/bin/bash', '-euo', 'pipefail'] - } - } -} -``` - -and running nextflow by setting the profile to `fixed_conda`: - -``` -nextflow run epi2me-labs/wf-template \ - -c my_config.cfg \ - -profile fixed_conda \ - ... -``` - -## Updating the workflow - -Periodically when running the workflow, users may find that a message is displayed -indicating that an update to the workflow is available. - -To update the workflow simply run: - - nextflow pull epi2me-labs/wf-template - -## Building the docker container from source - -The docker image used for running the `wf-template` workflow is available on -[dockerhub](https://hub.docker.com/repository/docker/ontresearch/wf-template). -The image is built from the Dockerfile present in the git repository. Users -wishing to modify and build the image can do so with: - -``` -CONTAINER_TAG=ontresearch/wf-template:latest - -git clone https://github.com/epi2me-labs/wf-template -cd wf-template - -docker build \ - -t ${CONTAINER_TAG} -f Dockerfile \ - --build-arg BASEIMAGE=ontresearch/base-workflow-image:v0.1.0 \ - . -``` - -In order to run the workflow with this new image it is required to give -`nextflow` the `--wfversion` parameter: - -``` -nextflow run epi2me-labs/wf-template \ - --wfversion latest -``` - ## Useful links * [nextflow](https://www.nextflow.io/) diff --git a/bin/check_sample_sheet.py b/bin/check_sample_sheet.py new file mode 100755 index 0000000..ae8fb76 --- /dev/null +++ b/bin/check_sample_sheet.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +"""Script to check that sample sheet is well-formatted.""" +import argparse +import sys + +import pandas as pd + + +def main(): + """Run entry point.""" + parser = argparse.ArgumentParser() + parser.add_argument('sample_sheet') + parser.add_argument('output') + args = parser.parse_args() + + try: + samples = pd.read_csv(args.sample_sheet, sep=None) + if 'alias' in samples.columns: + if 'sample_name' in samples.columns: + sys.stderr.write( + "Warning: sample sheet contains both 'alias' and " + 'sample_name, using the former.') + samples['sample_name'] = samples['alias'] + if 'barcode' not in samples.columns \ + or 'sample_name' not in samples.columns: + raise IOError() + except Exception: + raise IOError( + "Could not parse sample sheet, it must contain two columns " + "named 'barcode' and 'sample_name' or 'alias'.") + # check duplicates + dup_bc = samples['barcode'].duplicated() + dup_sample = samples['sample_name'].duplicated() + if any(dup_bc) or any(dup_sample): + raise IOError( + "Sample sheet contains duplicate values.") + samples.to_csv(args.output, sep=",", index=False) + + +if __name__ == '__main__': + main() diff --git a/bin/conda_versions.py b/bin/conda_versions.py deleted file mode 100644 index 92be7d0..0000000 --- a/bin/conda_versions.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Scrape versions of conda packages.""" - -from collections import namedtuple -import subprocess - - -try: - import pandas as pd -except ImportError: - pass - - -PackageInfo = namedtuple( - 'PackageInfo', ('Name', 'Version', 'Build', 'Channel')) - - -def scrape_data(as_dataframe=False, include=None): - """Return versions of conda packages in base environment.""" - cmd = """ -. ~/conda/etc/profile.d/mamba.sh; -micromamba activate; -micromamba list; - """ - proc = subprocess.run(cmd, shell=True, check=True, capture_output=True) - versions = dict() - for line in proc.stdout.splitlines()[3:]: - items = line.decode().strip().split() - if len(items) == 3: - # sometimes channel isn't listed :/ - items.append("") - if include is None or items[0] in include: - versions[items[0]] = PackageInfo(*items) - if as_dataframe: - versions = pd.DataFrame.from_records( - list(versions.values()), - columns=PackageInfo._fields) - return versions diff --git a/bin/report.py b/bin/report.py index 259d286..42955aa 100755 --- a/bin/report.py +++ b/bin/report.py @@ -4,8 +4,8 @@ import argparse from aplanat.components import fastcat +from aplanat.components import simple as scomponents from aplanat.report import WFReport -import conda_versions def main(): @@ -13,6 +13,9 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("report", help="Report output file") parser.add_argument("summaries", nargs='+', help="Read summary file.") + parser.add_argument( + "--versions", required=True, + help="directory containing CSVs containing name,version.") parser.add_argument( "--revision", default='unknown', help="git branch/tag of the executed workflow") @@ -27,17 +30,8 @@ def main(): report.add_section( section=fastcat.full_report(args.summaries)) - - section = report.add_section() - section.markdown(''' -### Software versions -The table below highlights versions of key software used within the analysis. -''') - req = [ - 'python', 'aplanat', 'pysam', 'fastcat'] - versions = conda_versions.scrape_data( - as_dataframe=True, include=req) - section.table(versions[['Name', 'Version', 'Build']], index=False) + report.add_section( + section=scomponents.version_table(args.versions)) # write report report.write(args.report) diff --git a/environment.yaml b/environment.yaml index 9cfcee2..24d7e5f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -6,6 +6,6 @@ channels: - defaults dependencies: - python==3.8.* - - aplanat >=0.3.5 + - aplanat >=0.5.0 - pysam - fastcat diff --git a/lib/fastqingress.nf b/lib/fastqingress.nf new file mode 100644 index 0000000..30713a7 --- /dev/null +++ b/lib/fastqingress.nf @@ -0,0 +1,170 @@ + +process checkSampleSheet { + label "artic" + cpus 1 + input: + file "sample_sheet.txt" + output: + file "samples.txt" + """ + check_sample_sheet.py sample_sheet.txt samples.txt + """ +} + + +/** + * Load a sample sheet into a Nextflow channel to map barcodes + * to sample names. + * + * @param samples CSV file according to MinKNOW sample sheet specification + * @return A Nextflow Channel of tuples (barcode, sample name) + */ +def check_sample_sheet(samples) +{ + println("Checking sample sheet.") + sample_sheet = Channel.fromPath(samples, checkIfExists: true) + sample_sheet = checkSampleSheet(sample_sheet) + .splitCsv(header: true) + .map { row -> tuple(row.barcode, row.sample_name) } + return sample_sheet +} + + +/** + * Find fastq data using various globs. Wrapper around Nextflow `file` + * method. + * + * @param patten glob pattern for top level input folder. + * @param maxdepth maximum depth to traverse + * @return list of files. + */ +def find_fastq(pattern, maxdepth) +{ + files = [] + extensions = ["fastq", "fastq.gz", "fq", "fq.gz"] + for (ext in extensions) { + files += file("${pattern}/*.${ext}", type: 'file', maxdepth: maxdepth) + } + return files +} + + +/** + * Rework EPI2ME flattened directory structure into standard form + * files are matched on barcode\d+ and moved into corresponding + * subdirectories ready for processing. + * + * @param input_folder Top-level input directory. + * @param output_folder Top-level output_directory. + * @return A File object representating the staging directory created + * under output_folder + */ +def sanitize_fastq(input_folder, output_folder) +{ + println("Running sanitization.") + println(" - Moving files: ${input_folder} -> ${output_folder}") + staging = new File(output_folder) + staging.mkdirs() + files = find_fastq("${input_folder}/**/", 1) + for (fastq in files) { + fname = fastq.getFileName() + // find barcode + pattern = ~/barcode\d+/ + matcher = fname =~ pattern + if (!matcher.find()) { + // not barcoded - leave alone + fastq.renameTo("${staging}/${fname}") + } else { + bc_dir = new File("${staging}/${matcher[0]}") + bc_dir.mkdirs() + fastq.renameTo("${staging}/${matcher[0]}/${fname}") + } + } + println(" - Finished sanitization.") + return staging +} + + +/** + * Resolves input folder containing barcode subdirectories + * or a flat set of fastq data to a Nextflow Channel. Removes barcode + * directories with no fastq files. + * + * @param input_folder Top level input folder to locate fastq data + * @param sample_sheet List of tuples mapping barcode to sample name + * or a simple string for non-multiplexed data. + * @return Channel of tuples (path, sample_name) + */ +def resolve_barcode_structure(input_folder, sample_sheet) +{ + println("Checking input directory structure.") + barcode_dirs = file("$input_folder/barcode*", type: 'dir', maxdepth: 1) + not_barcoded = find_fastq("$input_folder/", 1) + samples = null + if (barcode_dirs) { + println(" - Found barcode directories") + // remove empty barcode_dirs + valid_barcode_dirs = [] + invalid_barcode_dirs = [] + for (d in barcode_dirs) { + if(!find_fastq(d, 1)) { + invalid_barcode_dirs << d + } else { + valid_barcode_dirs << d + } + } + if (invalid_barcode_dirs.size() > 0) { + println(" - Some barcode directories did not contain .fastq(.gz) files:") + for (d in invalid_barcode_dirs) { + println(" - ${d}") + } + } + // link sample names to barcode through sample sheet + if (!sample_sheet) { + sample_sheet = Channel + .fromPath(valid_barcode_dirs) + .filter(~/.*barcode[0-9]{1,3}$/) // up to 192 + .map { path -> tuple(path.baseName, path.baseName) } + } + samples = Channel + .fromPath(valid_barcode_dirs) + .filter(~/.*barcode[0-9]{1,3}$/) // up to 192 + .map { path -> tuple(path.baseName, path) } + .join(sample_sheet) + .map { barcode, path, sample -> tuple(path, sample) } + } else if (not_barcoded) { + println(" - Found fastq files, assuming single sample") + sample = (sample_sheet == null) ? "unknown" : sample_sheet + samples = Channel + .fromPath(input_folder, type: 'dir', maxDepth:1) + .map { path -> tuple(path, sample) } + } + return samples +} + + +/** + * Take an input directory and sample sheet to return a channel of + * named samples. + * + * @param input_folder Top level input folder to locate fastq data + * @param sample_sheet List of tuples mapping barcode to sample name + * or a simple string for non-multiplexed data. + * @return Channel of tuples (path, sample_name) + */ +def fastq_ingress(input_folder, output_folder, samples, sanitize) +{ + // EPI2ME harness + if (sanitize) { + staging = "${output_folder}/staging" + input_folder = sanitize_fastq(input_folder, staging) + } + // check sample sheet + sample_sheet = null + if (samples) { + sample_sheet = check_sample_sheet(samples) + } + // resolve whether we have demultiplexed data or single sample + data = resolve_barcode_structure(input_folder, sample_sheet) + return data +} diff --git a/main.nf b/main.nf index 9d5ef80..6b94aec 100644 --- a/main.nf +++ b/main.nf @@ -12,6 +12,7 @@ nextflow.enable.dsl = 2 +include { fastq_ingress } from './lib/fastqingress' def helpMessage(){ log.info """ @@ -21,7 +22,9 @@ Usage: nextflow run epi2melabs/wf-template [options] Script Options: - --fastq DIR Path to directory containing FASTQ files (required) + --fastq DIR Path to FASTQ directory (required) + --samples FILE CSV file with columns named `barcode` and `sample_name` + (or simply a sample name for non-multiplexed data). --out_dir DIR Path for output (default: $params.out_dir) """ } @@ -33,24 +36,37 @@ process summariseReads { label "pysam" cpus 1 input: - file "input" + tuple path(directory), val(sample_name) output: - file "seqs.txt" + path "${sample_name}.stats" shell: """ - fastcat -r seqs.txt input/*.fastq* > /dev/null + fastcat -s ${sample_name} -r ${sample_name}.stats -x ${directory} > /dev/null """ } +process getVersions { + label "pysam" + cpus 1 + output: + path "versions.txt" + script: + """ + python -c "import pysam; print(f'pysam,{pysam.__version__}')" >> versions.txt + fastcat --version | sed 's/^/fastcat,/' >> versions.txt + """ +} + process makeReport { label "pysam" input: - file "seqs.txt" + path "seqs.txt" + path "versions/*" output: - file "wf-template-report.html" + path "wf-template-report.html" """ - report.py wf-template-report.html seqs.txt + report.py wf-template-report.html --versions versions seqs.txt """ } @@ -63,9 +79,9 @@ process output { label "pysam" publishDir "${params.out_dir}", mode: 'copy', pattern: "*" input: - file fname + path fname output: - file fname + path fname """ echo "Writing output files" """ @@ -78,7 +94,8 @@ workflow pipeline { reads main: summary = summariseReads(reads) - report = makeReport(summary) + software_versions = getVersions() + report = makeReport(summary, software_versions.collect()) emit: summary.concat(report) } @@ -98,12 +115,9 @@ workflow { exit 1 } - reads = file("$params.fastq/*.fastq*", type: 'file', maxdepth: 1) - if (reads) { - reads = Channel.fromPath(params.fastq, type: 'dir', checkIfExists: true) - results = pipeline(reads) - output(results) - } else { - println("No .fastq(.gz) files found under `${params.fastq}`.") - } + samples = fastq_ingress( + params.fastq, params.out_dir, params.samples, params.sanitize_fastq) + + results = pipeline(samples) + output(results) } diff --git a/nextflow.config b/nextflow.config index 2289286..f6f677c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -14,6 +14,8 @@ params { help = false fastq = null out_dir = "output" + samples = null + sanitize_fastq = false wfversion = "v0.0.6" aws_image_prefix = null aws_queue = null @@ -53,7 +55,7 @@ profiles { } process { withLabel:pysam { - conda = "environment.yaml" + conda = "${projectDir}/environment.yaml" } shell = ['/bin/bash', '-euo', 'pipefail'] }