Merge branch 'dev' of workflow-containers/wf-template into dev

.gitlab-ci.yml

@@ -1,4 +1,3 @@
-
 stages:
     - build_and_run
     - release
@@ -25,11 +24,14 @@ conda-run:
         - *install-nextflow
         - *install-conda
     script:
-        - ./nextflow run workflow.nf
+        - OUTPUT=${CI_PROJECT_NAME}
+        - ./nextflow run main.nf
               -w ${OUTPUT}/workspace
               -profile conda
-              --reads test_data/reads.fq.gz 
+              --fastq test_data/reads.fq.gz 
               --out_dir ${OUTPUT}
+    only:
+        - branches
 
 
 build-image:
@@ -43,11 +45,12 @@ build-image:
         - docker build --no-cache -t "${TAG}" -f Dockerfile . --build-arg BASEIMAGE=${BASEIMAGE}
         # run letting nextflow orchestrate the containers
         - docker tag "${TAG}" "${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:latest"
-        - OUTPUT="template-workflow"
-        - ./nextflow run workflow.nf
+        - OUTPUT=${CI_PROJECT_NAME}
+        - ./nextflow run main.nf
               -w ${OUTPUT}/workspace
               -profile standard
-              --reads test_data/reads.fq.gz 
+              --wfversion latest
+              --fastq test_data/reads.fq.gz 
               --out_dir ${OUTPUT}
         # push
         - if [[ ${CI_COMMIT_BRANCH} == 'dev' ]]; then
@@ -61,7 +64,7 @@ build-image:
     artifacts:
         paths:
             # Add the output directory for the test
-            - "template-workflow"
+            - ${CI_PROJECT_NAME}
         expire_in: 1 day
 
 
@@ -95,22 +98,26 @@ release-hub:
     script:
         - echo ${DOCKERHUB_TOKEN} | docker login --username epi2melabs --password-stdin
         - RELTAG="${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:${CI_COMMIT_TAG}"
+        - LATEST="${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:latest"
         - echo "Pushing ${RELTAG}"
         - docker tag ${SHATAG} ${RELTAG}
         - docker push ${RELTAG}
+        - docker tag ${SHATAG} ${LATEST}
+        - docker push ${LATEST}
 
 
 # Send all tags matching vX.Y.Z to github (code and release)
 push-github:
-    stage: deploy
+    stage: release
+    image: $GH_PUSH_IMAGE
     before_script:
         - apt-get update -qq && apt-get install -y -qq
           git python3-all-dev git-lfs python3-venv
         - mkdir ~/.ssh/
-        - cp $RESEARCH_BOT_GH_KEY ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
+        - cp $LABS_BOT_GH_KEY ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
         - echo -e "Host github.com\n\tStrictHostKeyChecking no\n\tHostname ssh.github.com\n\tPort 443\n\n" > ~/.ssh/config
-        - git config --global user.email "research.git@nanoporetech.com"
-        - git config --global user.name "ontresearch"
+        - git config --global user.email "epi2melabs@nanoporetech.com"
+        - git config --global user.name "epi2melabs-bot"
     script:
         # Push master and tag to github
         - git remote add ont ${CI_REPOSITORY_URL} || true
@@ -127,7 +134,7 @@ push-github:
         - source release_env/bin/activate
         - pip install pip --upgrade
         - pip install git+https://github.com/epi2me-labs/github_release.git
-        - github_release ${CI_PROJECT_NAME} ${CI_COMMIT_TAG} CHANGELOG.md ${RESEARCH_BOT_GH_TOKEN}
+        - github_release ${CI_PROJECT_NAME} ${CI_COMMIT_TAG} CHANGELOG.md ${LABS_BOT_GH_TOKEN}
     only:
         - /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/
     except:

README.md

@@ -40,7 +40,7 @@ OUTPUT=workflow-output
 nextflow run main.nf \
     -w ${OUTPUT}/workspace \
     -profile standard \
-    --reads test_data/reads.fq.gz \
+    --fastq test_data/reads.fq.gz \
     --out_dir ${OUTPUT}
 ```
 
@@ -59,7 +59,7 @@ OUTPUT=workflow-output
 nextflow run main.nf \
     -w ${OUTPUT}/workspace \
     -profile conda \
-    --reads test_data/reads.fq.gz \
+    --fastq test_data/reads.fq.gz \
     --out_dir ${OUTPUT}
 ```
 

bin/read_lengths.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import argparse
+import pysam
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('fasta')
+    parser.add_argument('output')
+    args = parser.parse_args()
+
+    with open(args.output, 'w') as fh:
+        for rec in pysam.FastxFile(args.fasta):
+            fh.write("{}\t{}\n".format(rec.name, len(rec.sequence)))
+
+if __name__ == '__main__':
+    main()

main.nf

@@ -12,21 +12,18 @@
 
 nextflow.enable.dsl = 2
 
-params.help = ""
 
-if(params.help) {
-    log.info ''
-    log.info 'Workflow template'
-    log.info ''
-    log.info 'Usage: '
-    log.info '    nextflow run workflow.nf [options]'
-    log.info ''
-    log.info 'Script Options: '
-    log.info '    --fastq        FILE    Path to FASTQ file'
-    log.info '    --out_dir      DIR     Path for output'
-    log.info ''
+def helpMessage(){
+    log.info """
+Workflow template'
 
-    return
+Usage:
+    nextflow run epi2melabs/workflow-template [options]
+
+Script Options:
+    --fastq        FILE    Path to FASTQ file (required)
+    --out_dir      DIR     Path for output (default: $params.out_dir)
+"""
 }
 
 
@@ -39,11 +36,8 @@ process readSeqs {
         file "seqs.txt"
 
     """
-    #!/usr/bin/env python
-    import pysam
-    with open("seqs.txt", 'w') as fh:
-        for rec in pysam.FastxFile("$reads"):
-            fh.write("{}\\t{}\\n".format(rec.name, len(rec.sequence)))
+    read_lengths.py $reads seqs.txt
+    sleep 60
     """
 }
 
@@ -78,7 +72,21 @@ workflow pipeline {
 
 // entrypoint workflow
 workflow {
-    reads = channel.fromPath(params.reads, checkIfExists:true)
+
+    if (params.help) {
+        helpMessage()
+        exit 1
+    }
+
+    if (!params.fastq) {
+        helpMessage()
+        println("")
+        println("`--fastq` is required")
+        exit 1
+    }
+
+
+    reads = channel.fromPath(params.fastq, checkIfExists:true)
     results = pipeline(reads)
     output(results)
 }

nextflow.config

@@ -10,6 +10,20 @@
 // for further help editing this file.
 
 
+params {
+    help = false
+    out_dir = "output"
+    wfversion = "v0.0.1"
+}
+
+
+executor {
+    $local {
+        cpus = 4
+        memory = "8 GB"
+    }
+}
+
 profiles {
     // the "standard" profile is used implicitely by nextflow
     // if no other profile is given on the CLI
@@ -22,7 +36,7 @@ profiles {
         }
         process {
             withLabel:pysam {
-                container = 'ontresearch/workflow-template:latest'
+                container = "ontresearch/workflow-template:${params.wfversion}"
             }
             shell = ['/bin/bash', '-euo', 'pipefail']
         }
@@ -45,3 +59,21 @@ profiles {
         }
     }
 }
+
+
+timeline {
+  enabled = true
+  file = "${params.out_dir}/execution_timeline.html"
+}
+report {
+  enabled = true
+  file = "${params.out_dir}/execution_report.html"
+}
+trace {
+  enabled = true
+  file = "${params.out_dir}/execution_trace.txt"
+}
+dag {
+  enabled = true
+  file = "${params.out_dir}/pipeline_dag.svg"
+}