From dd2c1efd91c90f8b1ac37774c5b5f0dd854b1725 Mon Sep 17 00:00:00 2001 From: Chris Wright Date: Fri, 29 Jan 2021 18:05:35 +0000 Subject: [PATCH] Condaenv --- .gitlab-ci.yml | 28 ++++++++++++++++++++++++++-- Dockerfile | 10 +++++----- README.md | 43 +++++++++++++++++++++++++++++++++++++++---- environment.yaml | 7 +++++++ nextflow.config | 25 +++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 environment.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7739ea..6f4aeeb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,12 +7,36 @@ image: ${IMAGE} variables: BASEIMAGE: ${CI_REGISTRY}/${CURRENT_TEMPLATE_IMAGE} +.install-nextflow: &install-nextflow | + wget -qO- https://get.nextflow.io | bash + +.install-conda: &install-conda | + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh -b -p ./miniconda + source ./miniconda/bin/activate + conda init + + +conda-run: + image: ubuntu:20.04 + stage: build_and_run + before_script: + - apt update && apt install -y wget default-jre + - *install-nextflow + - *install-conda + script: + - ./nextflow run workflow.nf + -w ${OUTPUT}/workspace + -profile conda + --reads test_data/reads.fq.gz + --out_dir ${OUTPUT} + + build-image: stage: build_and_run - variables: before_script: - apk add wget openjdk11 bash - - wget -qO- https://get.nextflow.io | bash + - *install-nextflow script: - echo ${CI_BUILD_TOKEN} | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} - TAG="${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" diff --git a/Dockerfile b/Dockerfile index fedf94c..a024f43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ ARG BASEIMAGE=epi2melabs/base-workflow-image:latest FROM $BASEIMAGE -# Minimal install for example purposes +# Minimal install for example purposes +COPY environment.yaml $HOME/environment.yaml RUN \ . $CONDA_DIR/etc/profile.d/mamba.sh \ - && micromamba activate \ - && micromamba install -y \ - pysam \ - -c anaconda -c conda-forge -c bioconda -q -y \ + && micromamba activate \ + && micromamba install --help \ + && micromamba install --file $HOME/environment.yaml \ && fix-permissions $CONDA_DIR \ && fix-permissions $HOME diff --git a/README.md b/README.md index 5952956..9b47574 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,19 @@ # Workflow template -This repository contains a nextflow workflow template and associated -container build. +This repository contains a Nextflow workflow template and associated Docker +container build. The workflow also supports using conda environments as an +alternative software isolation method to Docker. ## Quickstart +### Building the container + +> This step is not necessary if you intend to run the workflow using +> conda environments. + +The Docker container image can be built with the following command: + ```bash -# build the container CONTAINER_TAG=template-workflow docker build \ -t ${CONTAINER_TAG} -f Dockerfile \ @@ -16,11 +23,17 @@ docker build \ The `BASEIMAGE` argument here can be changed to use an alternative image. +### Running the workflow The template includes a simple workflow that outputs a file with the lengths of sequences contained in a .fastq.gz file. + +**Running the workflow with Docker containers** + +To run the workflow using Docker containers supply the `-profile standard` +argument to `nextflow run`: + ``` -# run the pipeline with the test data OUTPUT=template-workflow nextflow run workflow.nf \ -w ${OUTPUT}/workspace \ @@ -32,3 +45,25 @@ nextflow run workflow.nf \ The output of the pipeline will be found in `./template-workflow` for the above example. This directory contains the nextflow working directories alongside the two primary outputs of the pipeline. + +**Using conda environments** + +To run the workflow backed by conda environments, simply provide the +`-profile conda` argument to `nextflow run`. + +``` +# run the pipeline with the test data +OUTPUT=template-workflow +nextflow run workflow.nf \ + -w ${OUTPUT}/workspace \ + -profile conda \ + --reads test_data/reads.fq.gz \ + --out_dir ${OUTPUT} +``` + +This will create a conda environment with all required software within the +workspace directory. When running multiple analyses on distinct datasets +it may not be desirable to have Nextflow create a conda environment for each +analysis. To avoid the situation editing the file `nextflow.config` will +be necessary. Search for the term `cacheDir` and set this to a directory +where you wish the conda environment to be placed. diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..44ec003 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,7 @@ +name: epi2melabs-nf-template-workflow +channels: + - bioconda + - conda-forge + - defaults +dependencies: + - pysam diff --git a/nextflow.config b/nextflow.config index 5c4649d..c33a8f4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,3 +1,14 @@ +// +// Notes to End Users. +// +// The workflow should run without editing this configuration file, +// however there may be instances in which you wish to edit this +// file for compute performance or other reasons. Please see: +// +// https://nextflow.io/docs/latest/config.html#configuration +// +// for further help editing this file. + process { withLabel:containerCPU { @@ -16,4 +27,18 @@ profiles { runOptions = "--user \$(id -u):\$(id -g) --group-add 100" } } + + // profile using conda environments rather than docker + // containers + conda { + docker { + enabled = false + } + process { + conda = "environment.yaml" + } + conda { + cacheDir = "" + } + } }