Merge branch 'glue' into 'dev'

Glue

See merge request epi2melabs/workflows/wf-transcriptomes!90

.gitignore

@@ -3,7 +3,5 @@ nextflow
 template-workflow
 .*.swp
 .*.swo
-.DS_STORE
+*.pyc
-output/**
+*.pyo
-.idea/**
-**/__pycache__

.pre-commit-config.yaml

@@ -21,6 +21,7 @@ repos:
     rev: 3.7.9
     hooks:
       - id: flake8
+        pass_filenames: false
         additional_dependencies:
           - flake8-rst-docstrings
           - flake8-docstrings
@@ -31,4 +32,9 @@ repos:
           - flake8-builtins
           - flake8-absolute-import
           - flake8-print
-        entry: flake8 bin --import-order-style google --statistics
+        args: [
+            "bin",
+            "--import-order-style=google",
+            "--statistics",
+            "--max-line-length=88",
+        ]

README.md

@@ -6,6 +6,10 @@ It has been adapted from two existing Snakemake pipelines:
 * https://github.com/nanoporetech/pipeline-nanopore-ref-isoforms
 * https://github.com/nanoporetech/pipeline-nanopore-denovo-isoforms
 
+
+
+
+
 ## Introduction
 
 This workflow identifies RNA isoforms using either cDNA or direct RNA (dRNA) 
@@ -80,6 +84,10 @@ Differential gene expression is sensitive to the input data quantity and quality
 - Reference genome in fasta format (required for reference-based assembly).
 - Optional reference annotation in GFF2/3 format (required for differential expression analysis `--de_analysis`).
 - For fusion detection, JAFFAL reference files (see Quickstart) 
+
+
+
+
 ## Quickstart
 
 The workflow uses [nextflow](https://www.nextflow.io/) to manage compute and 
@@ -284,7 +292,11 @@ in `${out_dir}/jaffal_output_${sample_id}` you will find:
 * Nowicka, Malgorzata, and Mark D. Robinson. 2016. “DRIMSeq: A Dirichlet-Multinomial Framework for Multivariate Count Outcomes in Genomics [Version 2; Referees: 2 Approved].” F1000Research 5 (1356). https://doi.org/10.12688/f1000research.8900.2.
 * Patro, Robert, Geet Duggal, Michael I Love, Rafael A Irizarry, and Carl Kingsford. 2017. “Salmon Provides Fast and Bias-Aware Quantification of Transcript Expression.” Nature Methods 14 (March). https://doi.org/10.1038/nmeth.4197.
 * Robinson, Mark D, Davis J McCarthy, and Gordon K Smyth. 2010. “EdgeR: A Bioconductor Package for Differential Expression Analysis of Digital Gene Expression Data.” Bioinformatics 26 (1): 139–40.
-* Love, Michael I., et al. Swimming Downstream: Statistical Analysis of Differential Transcript Usage Following Salmon Quantification. 7:952, F1000Research, 14 Sept. 2018. f1000research.com, https://f1000research.com/articles/7-952## Useful links
+* Love, Michael I., et al. Swimming Downstream: Statistical Analysis of Differential Transcript Usage Following Salmon Quantification. 7:952, F1000Research, 14 Sept. 2018. f1000research.com, https://f1000research.com/articles/7-952
+
+
+
+## Useful links
 
 * [nextflow](https://www.nextflow.io/)
 * [docker](https://www.docker.com/products/docker-desktop)

bin/check_sample_sheet.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""Script to check that sample sheet is well-formatted."""
-import argparse
-import sys
-
-import pandas as pd
-
-
-def main():
-    """Run entry point."""
-    parser = argparse.ArgumentParser()
-    parser.add_argument('sample_sheet')
-    parser.add_argument('output')
-    args = parser.parse_args()
-
-    try:
-        samples = pd.read_csv(args.sample_sheet, sep=None)
-        if 'alias' in samples.columns:
-            if 'sample_id' in samples.columns:
-                sys.stderr.write(
-                    "Warning: sample sheet contains both 'alias' and "
-                    'sample_id, using the former.')
-            samples['sample_id'] = samples['alias']
-        if not set(['sample_id', 'barcode']).intersection(samples.columns):
-            raise IOError()
-    except Exception:
-        raise IOError(
-            "Could not parse sample sheet, it must contain two columns "
-            "named 'barcode' and 'sample_id' or 'alias'.")
-    # check duplicates
-    dup_bc = samples['barcode'].duplicated()
-    dup_sample = samples['sample_id'].duplicated()
-    if any(dup_bc) or any(dup_sample):
-        raise IOError(
-            "Sample sheet contains duplicate values.")
-    samples.to_csv(args.output, sep=",", index=False)
-
-
-if __name__ == '__main__':
-    main()

bin/ping.py

@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-"""Send workflow ping."""
-
-import argparse
-import json
-import uuid
-
-from epi2melabs import ping
-
-
-def get_uuid(val):
-    """Construct UUID from string."""
-    return uuid.UUID(str(val))
-
-
-def main():
-    """Run the entry point."""
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--hostname", required=True, default=None,
-        help="ping some meta")
-    parser.add_argument(
-        "--opsys", required=True, default=None,
-        help="ping some meta")
-    parser.add_argument(
-        "--session", default=None,
-        help="ping some meta")
-    parser.add_argument(
-        "--message", required=True, default=None,
-        help="message to include in the ping")
-    parser.add_argument(
-        "--meta", default=None,
-        help="JSON file of metadata to be included in the ping")
-    parser.add_argument(
-        "--revision", default='unknown',
-        help="git branch/tag of the executed workflow")
-    parser.add_argument(
-        "--commit", default='unknown',
-        help="git commit of the executed workflow")
-    parser.add_argument(
-        "--disable", action='store_true',
-        help="Run the script but don't send the ping")
-    args = parser.parse_args()
-
-    meta = None
-    if args.meta:
-        with open(args.meta, "r") as json_file:
-            meta = json.load(json_file)
-
-    if not args.disable:
-        ping.Pingu(
-            get_uuid(args.session),
-            hostname=args.hostname,
-            opsys=args.opsys
-        ).send_workflow_ping(
-            workflow='wf-transcriptomes',
-            message=args.message,
-            revision=args.revision,
-            commit=args.commit,
-            meta=meta
-        )
-
-
-if __name__ == "__main__":
-    main()

bin/workflow-glue

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+"""Entry point for sc_tools (single_cell_tools)."""
+
+from workflow_glue import cli
+
+if __name__ == '__main__':
+    cli()

bin/workflow_glue/__init__.py

@@ -0,0 +1,62 @@
+"""Workflow Python code."""
+import argparse
+import glob
+import importlib
+import os
+
+from workflow_glue.util import _log_level, get_main_logger  # noqa: ABS101
+
+
+__version__ = "0.0.1"
+_package_name = "workflow_glue"
+
+
+def get_components():
+    """Find a list of workflow command scripts."""
+    path = os.path.dirname(os.path.abspath(__file__))
+    components = list()
+    for fname in glob.glob(os.path.join(path, "*.py")):
+        name = os.path.splitext(os.path.basename(fname))[0]
+        if name in ("__init__", "util"):
+            continue
+        mod = importlib.import_module(f"{_package_name}.{name}")
+        # if there's a main() and and argparser() that's good enough for us.
+        try:
+            req = "main", "argparser"
+            if all(callable(getattr(mod, x)) for x in req):
+                components.append(name)
+        except Exception:
+            pass
+    return components
+
+
+def cli():
+    """Run workflow entry points."""
+    parser = argparse.ArgumentParser(
+        'wf-glue',
+        parents=[_log_level()],
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument(
+        '-v', '--version', action='version',
+        version='%(prog)s {}'.format(__version__))
+
+    subparsers = parser.add_subparsers(
+        title='subcommands', description='valid commands',
+        help='additional help', dest='command')
+    subparsers.required = True
+
+    # all component demos, plus some others
+    components = [
+        f'{_package_name}.{comp}' for comp in get_components()]
+    for module in components:
+        mod = importlib.import_module(module)
+        p = subparsers.add_parser(
+            module.split(".")[-1], parents=[mod.argparser()])
+        p.set_defaults(func=mod.main)
+
+    logger = get_main_logger(_package_name)
+    args = parser.parse_args()
+
+    logger.info("Starting entrypoint.")
+    args.func(args)

bin/workflow_glue/check_sample_sheet.py

@@ -0,0 +1,81 @@
+"""Check if a sample sheet is valid."""
+import csv
+import sys
+
+from .util import get_named_logger, wf_parser  # noqa: ABS101
+
+
+def main(args):
+    """Run the entry point."""
+    logger = get_named_logger("checkSheet")
+
+    barcodes = []
+    aliases = []
+    types = []
+
+    try:
+        with open(args.sample_sheet, "r") as f:
+            csv_reader = csv.DictReader(f)
+            n_row = 0
+            for row in csv_reader:
+                n_row += 1
+                if n_row == 1:
+                    n_cols = len(row)
+                else:
+                    # check we got the same number of fields
+                    if len(row) != n_cols:
+                        raise ValueError(
+                            f"Unexpected number of cells in row number {n_row}."
+                        )
+                try:
+                    barcodes.append(row["barcode"])
+                except KeyError:
+                    sys.stdout.write("'barcode' column missing")
+                    exit()
+                try:
+                    aliases.append(row["alias"])
+                except KeyError:
+                    sys.stdout.write("'alias' column missing")
+                    exit()
+                try:
+                    types.append(row["type"])
+                except KeyError:
+                    pass
+    except Exception as e:
+        sys.stdout.write(f"Parsing error: {e}")
+        exit()
+
+    # check barcode and alias values are unique
+    if len(barcodes) > len(set(barcodes)):
+        sys.stdout.write("values in 'barcode' column not unique")
+        exit()
+    if len(aliases) > len(set(aliases)):
+        sys.stdout.write("values in 'alias' column not unique")
+        exit()
+
+    if types:
+        # check if "type" column has unexpected values
+        unexp_type_vals = set(types) - set(
+            [
+                "test_sample",
+                "positive_control",
+                "negative_control",
+                "no_template_control",
+            ]
+        )
+        if unexp_type_vals:
+            sys.stdout.write(
+                f"found unexpected values in 'type' column: {unexp_type_vals}. "
+                "allowed values are: `['test_sample', 'positive_control', "
+                "'negative_control', 'no_template_control']`"
+            )
+            exit()
+
+    logger.info(f"Checked sample sheet {args.sample_sheet}.")
+
+
+def argparser():
+    """Argument parser for entrypoint."""
+    parser = wf_parser("check_sample_sheet")
+    parser.add_argument("sample_sheet", help="Sample sheet to check")
+    return parser

bin/workflow_glue/compute_cluster_quality.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python
-
 """Generate cluster quality data."""
 
 # Adapted form script by Kristoffer Sahlin for
 # isONclust: https://github.com/ksahlin/isONclust
 
-import argparse
 from collections import defaultdict
 import math
 from pathlib import Path
@@ -19,9 +17,56 @@ import pysam
 from sklearn.metrics.cluster import (
     adjusted_rand_score, completeness_score,
     homogeneity_score, v_measure_score)
+
+from .util import wf_parser  # noqa: ABS101
+
 matplotlib.use('Agg')
 
 
+def argparser():
+    """Argument parser for entrypoint."""
+    parser = wf_parser("compute_cluster_quality")
+    parser.add_argument(
+        '--clusters',
+        type=str,
+        help='Inferred clusters (tsv file)')
+    parser.add_argument(
+        '--classes',
+        type=str,
+        help='A sorted and indexed bam file.')
+    parser.add_argument(
+        '--ctsv',
+        default=None,
+        type=str,
+        help='Write true classes in this TSV file.')
+    parser.add_argument(
+        '--simulated',
+        action="store_true",
+        help='Simulated data, we can simply read correct classes '
+             'from the ref field.')
+    parser.add_argument(
+        '--ont',
+        action="store_true",
+        help='ONT data, parsing accessions differently.')
+    parser.add_argument(
+        '--modified_ont',
+        action="store_true",
+        help='ONT data preprocessed accessions, parsing '
+             'accessions differently.')
+    parser.add_argument('--outfile', type=str, help='Output file with results')
+    parser.add_argument(
+        '--report',
+        type=str,
+        help='Output PDF file with report')
+    parser.add_argument('--sizes', type=str, help='Cluster sizes')
+    parser.add_argument(
+        '--raw_data_out',
+        type=str,
+        help='dir to save raw data for plotting')
+
+    return parser
+
+
 def parse_inferred_clusters_tsv(tsv_file, args):
     """parse_inferred_clusters_tsv."""
     infile = open(tsv_file, "r")
@@ -602,51 +647,3 @@ def main(args):
     plt.clf()
 
     pages.close()
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description="Align predicted transcripts to transcripts in ensembl "
-                    "reference data base.")
-    parser.add_argument(
-        '--clusters',
-        type=str,
-        help='Inferred clusters (tsv file)')
-    parser.add_argument(
-        '--classes',
-        type=str,
-        help='A sorted and indexed bam file.')
-    parser.add_argument(
-        '--ctsv',
-        default=None,
-        type=str,
-        help='Write true classes in this TSV file.')
-    parser.add_argument(
-        '--simulated',
-        action="store_true",
-        help='Simulated data, we can simply read correct classes '
-             'from the ref field.')
-    parser.add_argument(
-        '--ont',
-        action="store_true",
-        help='ONT data, parsing accessions differently.')
-    parser.add_argument(
-        '--modified_ont',
-        action="store_true",
-        help='ONT data preprocessed accessions, parsing '
-             'accessions differently.')
-    parser.add_argument('--outfile', type=str, help='Output file with results')
-    parser.add_argument(
-        '--report',
-        type=str,
-        help='Output PDF file with report')
-    parser.add_argument('--sizes', type=str, help='Cluster sizes')
-    parser.add_argument(
-        '--raw_data_out',
-        type=str,
-        help='dir to save raw data for plotting')
-    args = parser.parse_args()
-
-    sys.stdout("------------------------------------------------------------")
-    main(args)
-    sys.stdout("------------------------------------------------------------")

bin/workflow_glue/generate_pychopper_stats.py

@@ -3,20 +3,20 @@
 
 # -*- coding: utf-8 -*-
 
-import argparse
 import os
-import sys
 
 import pandas as pd
 
+from .util import wf_parser  # noqa: ABS101
 
-def parse_args(argv=sys.argv[1:]):
+
-    """Parse arguments."""
+def argparser():
-    description = """Script to run the isoform workflow """
+    """Argument parser for entrypoint."""
-    parser = argparse.ArgumentParser(description=description)
+    parser = wf_parser("generate_pychopper_stats")
     parser.add_argument("--data", required=True, help="")
     parser.add_argument("--output_dir", required=True, help="")
-    return parser.parse_args(argv)
+
+    return parser
 
 
 def generate_pychopper_stats(tsv, output):
@@ -37,7 +37,3 @@ def main(args):
     assert os.path.isfile(args.data)
     assert os.path.isdir(args.output_dir)
     generate_pychopper_stats(tsv=args.data, output=args.output_dir)
-
-
-if __name__ == '__main__':
-    main(args=parse_args())

bin/workflow_glue/generate_tracking_summary.py

@@ -1,21 +1,22 @@
 #!/usr/bin/env python
 """Generate per-transcript class sumarrarry files from gffcompare."""
 
-import argparse
 import os
 import sys
 
 import pandas as pd
 
+from .util import wf_parser  # noqa: ABS101
 
-def parse_args(argv=sys.argv[1:]):
+
-    """Parse arguments."""
+def argparser():
-    description = """Script to run the isoform workflow """
+    """Argument parser for entrypoint."""
-    parser = argparse.ArgumentParser(description=description)
+    parser = wf_parser("generate_tracking_summary")
     parser.add_argument("--tracking", required=True, help="")
     parser.add_argument("--output_dir", required=True, help="")
     parser.add_argument("--annotation", required=False, default=None, help="")
-    return parser.parse_args(argv)
+
+    return parser
 
 
 def generate_tracking_summary(tracking_file, output_dir, annotations=None):
@@ -63,7 +64,3 @@ def main(args):
         os.path.isfile(args.annotation)
     generate_tracking_summary(
         args.tracking, output_dir=args.output_dir, annotations=args.annotation)
-
-
-if __name__ == '__main__':
-    main(parse_args(sys.argv[1:]))

bin/workflow_glue/merge_count_tsvs.py

@@ -1,36 +1,40 @@
 #!/usr/bin/env python
 """Merge salmon output count files."""
 
-import argparse
 from functools import reduce
 
 import numpy as np
 import pandas as pd
 
-# Parse command line arguments:
+from .util import wf_parser  # noqa: ABS101
-parser = argparse.ArgumentParser(
+
-    description="""Merge tab separated files on a given field using pandas.""")
+
-parser.add_argument(
+def argparser():
+    """Argument parser for entrypoint."""
+    parser = wf_parser("merge_count_tsvs")
+    parser.add_argument(
         '-j', metavar='join', help="Join type (outer).", default="outer")
-parser.add_argument(
+    parser.add_argument(
         '-f', metavar='field',
         help="Join on this field (Reference).", default="Reference")
-parser.add_argument(
+    parser.add_argument(
         '-o', metavar='out_tsv',
         help="Output tsv (merge_tsvs.tsv).", default="merge_tsvs.tsv")
-parser.add_argument(
+    parser.add_argument(
         '-z', action="store_true",
         help="Fill NA values with zero.", default=False)
-parser.add_argument(
+    parser.add_argument(
         '-tpm', type=bool, default=False,
         help="TPM instead of counts")
-parser.add_argument(
+    parser.add_argument(
         '-tsvs', metavar='input_tsvs', nargs='*',
         help="Input tab separated files.")
 
-if __name__ == '__main__':
+    return parser
 
-    args = parser.parse_args()
+
+def main(args):
+    """Run entry point."""
     dfs = {x: pd.read_csv(x, sep="\t") for x in args.tsvs}
 
     ndfs = []

bin/workflow_glue/report.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 """Create workflow report."""
 
-import argparse
 from collections import Counter, defaultdict, OrderedDict
 import math
 import os
@@ -19,12 +18,66 @@ from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.palettes import Category10_10
 from bokeh.plotting import figure
 from bokeh.transform import dodge
-import de_plots
 import gffutils
 import numpy as np
 import pandas as pd
 import sigfig
 
+from . import de_plots  # noqa: ABS101
+from .util import wf_parser  # noqa: ABS101
+
+
+def argparser():
+    """Argument parser for entrypoint."""
+    parser = wf_parser("report")
+    parser.add_argument("--report", help="Report output file")
+    parser.add_argument("--summaries", nargs='+', help="Read summary file.")
+    parser.add_argument(
+        "--versions", required=True,
+        help="directory containing CSVs containing name,version.")
+    parser.add_argument(
+        "--params", default=None, required=True,
+        help="A JSON file containing the workflow parameter key/values")
+    parser.add_argument(
+        "--revision", default='unknown',
+        help="git branch/tag of the executed workflow")
+    parser.add_argument(
+        "--commit", default='unknown',
+        help="git commit of the executed workflow")
+    parser.add_argument(
+        "--alignment_stats", required=False, default=None, nargs='*',
+        help="TSV summary file of alignment statistics")
+    parser.add_argument(
+        "--gff_annotation", required=False, nargs='+',
+        help="transcriptome annotation gff file")
+    parser.add_argument(
+        "--gffcompare_dir", required=False, default=None, nargs='*',
+        help="gffcompare outout dir")
+    parser.add_argument(
+        "--pychop_report", required=False, default=None,
+        help="TSV summary file of pychopper statistics")
+    parser.add_argument(
+        "--sample_ids", required=True, nargs='+',
+        help="List of sample ids")
+    parser.add_argument(
+        "--isoform_table_nrows", required=False, type=int, default=5000,
+        help="Maximum rows to display in isoforms table")
+    parser.add_argument(
+        "--cluster_qc_dirs", required=False, type=str, default=None, nargs='*',
+        help="Directory with various cluster quality csvs")
+    parser.add_argument(
+        "--jaffal_csv", required=False, type=str, default=None,
+        help="Path to JAFFAL results csv")
+    parser.add_argument(
+        "--de_report", required=False, type=str, default=None,
+        help="Differential expression report optional")
+    parser.add_argument(
+        "--de_stats", required=False, type=str, default=None, nargs='*',
+        help="Differential expression report optional")
+    parser.add_argument('--denovo', dest='denovo', action='store_true')
+
+    return parser
+
 
 def _parse_stat_line(sl):
     """Parse a stats line."""
@@ -846,57 +899,8 @@ def de_section(report):
         report=report)
 
 
-def main():
+def main(args):
     """Run the entry point."""
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--report", help="Report output file")
-    parser.add_argument("--summaries", nargs='+', help="Read summary file.")
-    parser.add_argument(
-        "--versions", required=True,
-        help="directory containing CSVs containing name,version.")
-    parser.add_argument(
-        "--params", default=None, required=True,
-        help="A JSON file containing the workflow parameter key/values")
-    parser.add_argument(
-        "--revision", default='unknown',
-        help="git branch/tag of the executed workflow")
-    parser.add_argument(
-        "--commit", default='unknown',
-        help="git commit of the executed workflow")
-    parser.add_argument(
-        "--alignment_stats", required=False, default=None, nargs='*',
-        help="TSV summary file of alignment statistics")
-    parser.add_argument(
-        "--gff_annotation", required=False, nargs='+',
-        help="transcriptome annotation gff file")
-    parser.add_argument(
-        "--gffcompare_dir", required=False, default=None, nargs='*',
-        help="gffcompare outout dir")
-    parser.add_argument(
-        "--pychop_report", required=False, default=None,
-        help="TSV summary file of pychopper statistics")
-    parser.add_argument(
-        "--sample_ids", required=True, nargs='+',
-        help="List of sample ids")
-    parser.add_argument(
-        "--isoform_table_nrows", required=False, type=int, default=5000,
-        help="Maximum rows to display in isoforms table")
-    parser.add_argument(
-        "--cluster_qc_dirs", required=False, type=str, default=None, nargs='*',
-        help="Directory with various cluster quality csvs")
-    parser.add_argument(
-        "--jaffal_csv", required=False, type=str, default=None,
-        help="Path to JAFFAL results csv")
-    parser.add_argument(
-        "--de_report", required=False, type=str, default=None,
-        help="Differential expression report optional")
-    parser.add_argument(
-        "--de_stats", required=False, type=str, default=None, nargs='*',
-        help="Differential expression report optional")
-    parser.add_argument('--denovo', dest='denovo', action='store_true')
-
-    args = parser.parse_args()
-
     sample_ids = args.sample_ids
 
     report = WFReport(
@@ -952,7 +956,3 @@ def main():
         section=scomponents.params_table(args.params))
 
     report.write(args.report)
-
-
-if __name__ == "__main__":
-    main()

bin/workflow_glue/run_isonclust2.py

@@ -3,10 +3,23 @@
 from collections import OrderedDict
 from glob import glob
 from itertools import zip_longest
+import os
 from pathlib import Path
 import re
 import subprocess as sub
 
+from .util import wf_parser  # noqa: ABS101
+
+
+def argparser():
+    """Argument parser for entrypoint."""
+    parser = wf_parser("report")
+    parser.add_argument(
+        "--workdir", help="directory containing batches/ dir [CWD]",
+        default=Path())
+
+    return parser
+
 
 class Node:
     """Node."""
@@ -90,8 +103,9 @@ def build_job_tree():
     return job_tree, levels
 
 
-def main():
+def main(args):
     """Entry point."""
+    os.chdir(args.workdir)
     Path('clusters').mkdir(exist_ok=True)
     job_tree, levels = build_job_tree()
 
@@ -122,8 +136,3 @@ def main():
     sub.call((
         "ln -s `realpath clusters/isONcluster_{}.cer` "
         "isONcluster_ROOT.cer".format(n.Id)), shell=True)
-
-
-if __name__ == '__main__':
-    # The cwd should be the process dir that contains 'batches/'
-    main()

bin/workflow_glue/tests/__init__.py

@@ -0,0 +1 @@
+"""__init__.py for the tests."""

bin/workflow_glue/tests/test_test.py

@@ -0,0 +1,10 @@
+"""A dummy test."""
+
+import argparse
+
+from workflow_glue import report
+
+
+def test():
+    """Just showing that we can import using the workflow-glue."""
+    assert isinstance(report.argparser(), argparse.ArgumentParser)

bin/workflow_glue/util.py

@@ -0,0 +1,52 @@
+"""The odd helper function."""
+
+import argparse
+import logging
+
+_log_name = None
+
+
+def get_main_logger(name):
+    """Create the top-level logger."""
+    global _log_name
+    _log_name = name
+    logging.basicConfig(
+        format='[%(asctime)s - %(name)s] %(message)s',
+        datefmt='%H:%M:%S', level=logging.INFO)
+    return logging.getLogger(name)
+
+
+def get_named_logger(name):
+    """Create a logger with a name.
+
+    :param name: name of logger.
+    """
+    name = name.ljust(10)[:10]  # so logging is aligned
+    logger = logging.getLogger('{}.{}'.format(_log_name, name))
+    return logger
+
+
+def wf_parser(name):
+    """Make an argument parser for a workflow command."""
+    return argparse.ArgumentParser(
+        name,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        add_help=False)
+
+
+def _log_level():
+    """Parser to set logging level and acquire software version/commit."""
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=False)
+
+    modify_log_level = parser.add_mutually_exclusive_group()
+    modify_log_level.add_argument(
+        '--debug', action='store_const',
+        dest='log_level', const=logging.DEBUG, default=logging.INFO,
+        help='Verbose logging of debug information.')
+    modify_log_level.add_argument(
+        '--quiet', action='store_const',
+        dest='log_level', const=logging.WARNING, default=logging.INFO,
+        help='Minimal logging; warnings only.')
+
+    return parser

lib/Pinguscript.groovy

@@ -8,7 +8,7 @@ class Pinguscript {
         def msgId = UUID.randomUUID().toString()
         def hosthash = null
         try {
-            hosthash = InetAddress.getLocalHost().getHostName().md5()
+            hosthash = InetAddress.getLocalHost().getHostName()
         } catch(Exception e) {
             hosthash = "Unavailable"
         }
@@ -32,7 +32,7 @@ class Pinguscript {
         def meta = meta_json "error": errorMessage.toString(), "profile": profile.toString(),
             "agent": agent.toString()
         meta+=any_other_data
-        def ping_version = '2.0.1'
+        def ping_version = '2.0.2'
         def tracking_json = new JsonBuilder()
         def tracking_id = tracking_json "msg_id": msgId, "version": ping_version
         def data_json = new JsonBuilder()

lib/fastqingress.nf

@@ -32,7 +32,7 @@ process checkSampleSheet {
     output:
         file "samples.txt"
     """
-    check_sample_sheet.py sample_sheet.txt samples.txt
+    workflow-glue check_sample_sheet sample_sheet.txt samples.txt
     """
 }
 

main.nf

@@ -30,6 +30,7 @@ process summariseConcatReads {
         tuple val(meta.sample_id), path('*.stats'), emit: summary
     script:
     """
+
     fastcat -s ${meta.sample_id} -r ${meta.sample_id}.stats -x ${directory} >  ${meta.sample_id}.fastq
     """
 }
@@ -55,7 +56,7 @@ process getVersions {
     stringtie --version | sed 's/^/stringtie,/' >> versions.txt
     gffcompare --version | head -n 1 | sed 's/ /,/' >> versions.txt
     spoa --version | sed 's/^/spoa,/' >> versions.txt
-    isONclust2 version | sed 's/ version: /,/' >> versions.txt
+#     isONclust2 version | sed 's/ version: /,/' >> versions.txt
     """
 }
 
@@ -67,6 +68,8 @@ process getParams {
         path "params.json"
     script:
         def paramsJSON = new JsonBuilder(params).toPrettyString()
+        println('test')
+        println(params.workDir)
     """
     # Output nextflow params object to JSON
     echo '$paramsJSON' > params.json
@@ -91,7 +94,7 @@ process preprocess_reads {
         """
         pychopper -t ${params.threads} ${params.pychopper_opts} ${input_reads} ${sample_id}_full_length_reads.fastq
         mv pychopper.tsv ${sample_id}_pychopper.tsv
-        generate_pychopper_stats.py --data ${sample_id}_pychopper.tsv --output .
+        workflow-glue generate_pychopper_stats --data ${sample_id}_pychopper.tsv --output .
 
         # Add sample id column
         sed "1s/\$/\tsample_id/; 1 ! s/\$/\t${sample_id}/" ${sample_id}_pychopper.tsv > tmp
@@ -242,7 +245,7 @@ process run_gffcompare{
         gffcompare -o ${out_dir}/str_merged -r ${ref_annotation} \
             ${params.gffcompare_opts} ${query_annotation}
 
-        generate_tracking_summary.py --tracking $out_dir/str_merged.tracking \
+        workflow-glue generate_tracking_summary --tracking $out_dir/str_merged.tracking \
             --output_dir ${out_dir} --annotation ${ref_annotation}
 
         mv *.tmap $out_dir
@@ -350,7 +353,7 @@ process makeReport {
     else
         OPT_PC_REPORT="--pychop_report pychopper_report/*"
     fi
-    report.py --report $report_name \
+    workflow-glue report --report $report_name \
     --versions $versions \
     --params params.json \
     \$OPT_ALN \

nextflow.config

@@ -24,7 +24,6 @@ params {
     out_dir = "output"
     sample = null
     sample_sheet = null
-    wfversion = "v0.1.8"
     aws_image_prefix = null
     aws_queue = null
     process_label = "isoforms"
@@ -34,7 +33,7 @@ params {
     monochrome_logs = false
     validate_params = true
     show_hidden_params = false
-    schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wfversion,wf,process_label'
+    schema_ignore_params = 'show_hidden_params,validate_params,monochrome_logs,aws_queue,aws_image_prefix,wf,process_label'
 
     // Process cDNA reads using pychopper, turn off for direct RNA:
     direct_rna = false
@@ -189,14 +188,17 @@ profiles {
 
 timeline {
   enabled = true
+  overwrite = true
   file = "${params.out_dir}/execution/timeline.html"
 }
 report {
   enabled = true
+  overwrite = true
   file = "${params.out_dir}/execution/report.html"
 }
 trace {
   enabled = true
+  overwrite = true
   file = "${params.out_dir}/execution/trace.txt"
 }
 

nextflow_schema.json

@@ -339,11 +339,6 @@
             "type": "string",
             "hidden": true
         },
-        "wfversion": {
-            "type": "string",
-            "default": "v0.1.8",
-            "hidden": true
-        },
         "monochrome_logs": {
             "type": "boolean"
         },

subworkflows/denovo_assembly.nf

@@ -199,7 +199,7 @@ process clustering() {
         tuple val(sample_id), path('isONcluster_ROOT.cer'), emit: root_cluster
     script:
     """
-    run_isonclust2.py $sorted_batches
+    workflow-glue run_isonclust2 $sorted_batches
     """
 }
 
@@ -228,7 +228,7 @@ process cluster_quality() {
         samtools view -q 2 -F 2304 -b - |\
          samtools sort - -o $bam;
     samtools index $bam;
-    compute_cluster_quality.py --sizes $final_clusters_dir/clusters_info.tsv \
+    workflow-glue compute_cluster_quality --sizes $final_clusters_dir/clusters_info.tsv \
         --outfile ${qc_dir}/cluster_quality.csv --ont --clusters $final_clusters_dir/clusters.tsv \
         --classes $bam --report ${qc_dir}/cluster_quality.pdf --raw_data_out $qc_dir_raw
     """

subworkflows/differential_expression.nf

@@ -24,7 +24,7 @@ process mergeCounts {
     output:
         path "all_counts.tsv"
     """
-    merge_count_tsvs.py -z -o all_counts.tsv -tsvs ${counts}
+    workflow-glue merge_count_tsvs -z -o all_counts.tsv -tsvs ${counts}
     """
 }
 
@@ -35,7 +35,7 @@ process mergeTPM {
     output:
         path "tpm_counts.tsv"
     """
-    merge_count_tsvs.py -o tpm_counts.tsv -z -tpm True -tsvs $counts 
+    workflow-glue merge_count_tsvs -o tpm_counts.tsv -z -tpm True -tsvs $counts
     """
 }