#!/usr/bin/env python """Create workflow report.""" import argparse from collections import OrderedDict from aplanat import bars, lines from aplanat.components import fastcat from aplanat.components import simple as scomponents from aplanat.report import WFReport from aplanat.util import Colors from bokeh.layouts import gridplot from bokeh.models import ColumnDataSource from bokeh.palettes import Category10_10 from bokeh.plotting import figure from bokeh.transform import dodge import numpy as np import pandas as pd def simple_hbar(df, y, right, title="", color=Colors.cerulean, fig_kwargs={}, plot_kwargs={}): """Create a simple barplot. :param groups: the grouping variable (the x-axis values). :param values: the data for bars are drawn (the y-axis values). :param kwargs: kwargs for bokeh figure. Move to planat when it's working? """ defaults = { 'output_backend': 'webgl', 'plot_height': 300, 'plot_width': 600} defaults.update(fig_kwargs) p = figure(y_range=df[y], height=250, title=title, toolbar_location=None, tools="") plot_kwargs.update({'height': 0.2}) p.hbar(y=df[y], right=df[right], **plot_kwargs) return p def _parse_stat_line(sl): """Parse a stats line.""" res = {} tmp = sl.split(':')[1] tmp = tmp.split('|') res['sensitivity'] = float(tmp[0].strip()) res['precision'] = float(tmp[1].strip()) return res def _parse_matching_line(line): """Parse a metching line.""" tmp = line.split(':')[1].strip() return int(tmp) def _parse_mn_line(line): """Parse a miss or novel line.""" res = {} tmp = line.split(':')[1].strip() tmp = tmp.split('/') res['value'] = int(tmp[0]) tmp = tmp[1].split('(') res['value_total'] = int(tmp[0].strip()) res['percent'] = float(tmp[1].split('%)')[0]) return res def _parse_total_line(line): """Parse a total line.""" res = {} tmp = line.split(':')[1].strip() tmp = tmp.split('in') res['transcripts'] = int(tmp[0].strip()) tmp = tmp[1].split('loci') res['loci'] = int(tmp[0].strip()) tmp = int(tmp[1].split('(')[1].split(' ')[0]) res['me_transcripts'] = tmp return res def parse_gffcmp_stats(txt): """Parse a gffcompare stats file. :param txt: Path to the gffcompare stats file. :returns: Return as tuple of dataframes containing: perfromance statistics, match statistics, miss statistics, novel statistics, total statistics. :rtype: tuple """ sensitivity = [] precision = [] level = [] matching = OrderedDict() missed_level = [] missed = [] missed_total = [] missed_percent = [] novel_level = [] novel = [] novel_total = [] novel_percent = [] total_target = [] total_loci = [] total_transcripts = [] total_multiexonic = [] fh = open(txt, 'r') for line in fh: line = line.strip() if len(line) == 0: continue # Parse totals: if line.startswith('# Query mRNAs'): total_target.append('Query') r = _parse_total_line(line) total_loci.append(r['loci']) total_transcripts.append(r['transcripts']) total_multiexonic.append(r['me_transcripts']) if line.startswith('# Reference mRNAs '): total_target.append('Reference') r = _parse_total_line(line) total_loci.append(r['loci']) total_transcripts.append(r['transcripts']) total_multiexonic.append(r['me_transcripts']) # Parse basic statistics: if line.startswith('Base level'): st = _parse_stat_line(line) level.append('Base') sensitivity.append(st['sensitivity']) precision.append(st['precision']) if line.startswith('Exon level'): st = _parse_stat_line(line) level.append('Exon') sensitivity.append(st['sensitivity']) precision.append(st['precision']) if line.startswith('Intron level'): st = _parse_stat_line(line) level.append('Intron') sensitivity.append(st['sensitivity']) precision.append(st['precision']) if line.startswith('Intron chain level'): st = _parse_stat_line(line) level.append('Intron chain') sensitivity.append(st['sensitivity']) precision.append(st['precision']) if line.startswith('Transcript level'): st = _parse_stat_line(line) level.append('Transcript') sensitivity.append(st['sensitivity']) precision.append(st['precision']) if line.startswith('Locus level'): st = _parse_stat_line(line) level.append('Locus') sensitivity.append(st['sensitivity']) precision.append(st['precision']) # Parse match statistics: if line.startswith('Matching intron chains'): m = _parse_matching_line(line) matching['Intron chains'] = [m] if line.startswith('Matching transcripts'): m = _parse_matching_line(line) matching['Transcripts'] = [m] if line.startswith('Matching loci'): m = _parse_matching_line(line) matching['Loci'] = [m] # Parse missing statistics: if line.startswith('Missed exons'): missed_level.append('Exons') r = _parse_mn_line(line) missed.append(r['value']) missed_total.append(r['value_total']) missed_percent.append(r['percent']) if line.startswith('Missed introns'): missed_level.append('Introns') r = _parse_mn_line(line) missed.append(r['value']) missed_total.append(r['value_total']) missed_percent.append(r['percent']) if line.startswith('Missed loci'): missed_level.append('Loci') r = _parse_mn_line(line) missed.append(r['value']) missed_total.append(r['value_total']) missed_percent.append(r['percent']) # Parse novel statistics: if line.startswith('Novel exons'): novel_level.append('Exons') r = _parse_mn_line(line) novel.append(r['value']) novel_total.append(r['value_total']) novel_percent.append(r['percent']) if line.startswith('Novel introns'): novel_level.append('Introns') r = _parse_mn_line(line) novel.append(r['value']) novel_total.append(r['value_total']) novel_percent.append(r['percent']) if line.startswith('Novel loci'): novel_level.append('Loci') r = _parse_mn_line(line) novel.append(r['value']) novel_total.append(r['value_total']) novel_percent.append(r['percent']) fh.close() df_stats = pd.DataFrame(OrderedDict( [('Sensitivity', sensitivity), ('Precision', precision)]), index=level) df_match = pd.DataFrame(matching, index=['Matching']) df_miss = pd.DataFrame( OrderedDict( [('Total', missed_total), ('Missed', missed), ('Percent missed', missed_percent)]), index=missed_level) df_novel = pd.DataFrame( OrderedDict( [('Total', novel_total), ('Novel', novel), ('Percent novel', novel_percent)]), index=novel_level) df_total = pd.DataFrame(OrderedDict( [('Loci', total_loci), ('Transcripts', total_transcripts), ('Multiexonic', total_multiexonic)]), index=total_target) return df_stats, df_match, df_miss, df_novel, df_total def grouped_bar(df, title=""): """Create grouped bar plot from pandas dataframe. :param pandas.DataFrame Index: str: the x group labels - groups cluserted using these Columns: numeric: sub-groups of data - each sub group has same colour :returns bokaoh.plotting.figure instance """ min_ = 0 max_ = df.to_numpy().max() max_ = max_ + (max_ * 0.3) # Add some padding at top of plot for legends yrange = int(min_), int(max_) df['x_groups'] = df.index df = df.reset_index(drop=True) source = ColumnDataSource(data=df) p = figure(x_range=df['x_groups'], y_range=yrange, height=250, title=title, toolbar_location=None, tools="") i = 0 # Use the dodge method to plot groups of bars # https://docs.bokeh.org/en/latest/docs/user_guide/categorical.html dodge_range = (-0.25, 0.25) current_dodge = dodge_range[0] dodge_increment = abs(dodge_range[0] - dodge_range[1]) \ / (len(df.columns) - 1) for col in df.columns: num_colors = df.shape[1] - 1 colors = list(zip(*[[Category10_10[x]] * (len(df.columns) - 1) for x in range(num_colors)])) colors = [item for sublist in colors for item in sublist] if col == 'x_groups': continue color = colors[i] i += 1 width = df.size / 60 p.vbar(x=dodge('x_groups', current_dodge, range=p.x_range), top=col, width=width, source=source, color=color, legend_label=col) current_dodge += dodge_increment p.x_range.range_padding = 0.1 p.xgrid.grid_line_color = None p.legend.location = "top_left" p.legend.orientation = "horizontal" return p def workflow_plots(report, df_aln_stats_file, gff_cmp_stats_file, gff_cmp_tracking_file): """Create various sections and plots in a WfReport. :param report: aplanat WFReport :param df_aln_stats_file: alignment stats. Output of `seqkit bam -s` :param gff_cmp_stats_file: gffcompare stats file :param gff_cmp_tracking_file: gffcompare tracking file :return: None """ df_aln_stats = pd.read_csv(df_aln_stats_file, sep='\t') df_aln_stats = df_aln_stats.select_dtypes([np.number]).dropna(axis=1) section = report.add_section() section.markdown(''' ### Read mapping summary Output of [seqkit](https://bioinf.shenwei.me/seqkit/) bam -s''') section.table(df_aln_stats) # Percentage primary and secondary mapping df_perc = df_aln_stats[['PrimAlnPerc', 'MultimapPerc']] bar_perc = bars.simple_bar( df_perc.columns.values, df_perc.iloc[0].values, title='% primary and multimapping reads', colors=Colors.cerulean) # Counts of read mapping class df_counts = df_aln_stats.drop(columns=['PrimAlnPerc', 'MultimapPerc']) bar_counts = bars.simple_bar( df_counts.columns.values, df_counts.iloc[0].values, title='Number of alignment records', colors=Colors.cerulean ) grid = gridplot([bar_perc, bar_counts], ncols=2, plot_width=400, plot_height=400) section.plot(grid) # If gffcompare has not been run, finish report here if not gff_cmp_stats_file or not gff_cmp_tracking_file: return stats, _, miss, novel, total = \ parse_gffcmp_stats(gff_cmp_stats_file) # Plot overview panel: section = report.add_section() section.markdown(''' ### Annotation summary The following plots summarize some of the output from [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) * **Totals**: Comparison of the number of stringtie-generated transcripts, multiexonic transcripts and loci (I'm not exactly sure what defines this class at the moment) between reference * **Performance**: How accurate are the query transcript annotations with respect to the reference at various levels. * **Missed**: Features present in the reference, but absent in the query * **Novel**: Features present in the query transcripts, but absent in the reference ''') bar_totals = grouped_bar(total, title="Totals") bar_performance = grouped_bar(stats, title="Performance") bar_missed = grouped_bar(miss, title="Missed") bar_novel = grouped_bar(novel, title="Novel") grid = gridplot([bar_totals, bar_performance, bar_missed, bar_novel], ncols=2, plot_width=400, plot_height=400) section.plot(grid) def fix_names(s): """Map trancript classification codes.""" names = { '=': 'ExactMatch:=', 'c': 'Contained:c', 'k': 'ReverseContained:k', 'm': 'RetainedIntron:m', 'n': 'PartRetainedIntron:n', 'j': 'PartialMatch:j', 'e': 'TransFragMatch:e', 's': 'OppositeMatch:s', 'o': 'OtherSameStrand:o', 'x': 'ExonicOpposite:o', 'y': 'RefInIntrons:y', 'p': 'PolymeraseRunon:p', 'r': 'Repeat:r', 'u': 'Intergenic:u', 'i': 'FullyIntronic:i', } return names[s] # Plot overlaps panel: section = report.add_section() section.markdown(''' ## Query transfrag class assignments The classes that are assinged by [gffcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml), which describe the relationship between query transfrag and the most similar reference transcript. [This diagram](https://ccb.jhu.edu/software/stringtie/ gffcompare_codes.png) illustrates the different classes. ''') tracking = pd.read_csv(gff_cmp_tracking_file, sep="\t", header=None, usecols=[0, 3], names=['Count', 'Overlaps']) tracking = tracking.groupby("Overlaps").count().reset_index() tracking = tracking.sort_values("Overlaps") tracking.Overlaps = tracking.Overlaps.apply(fix_names) tracking["Percent"] = tracking.Count * 100 / tracking.Count.sum() tracking_bar = simple_hbar( tracking, 'Overlaps', 'Count', title="totals") tracking_bar_perc = simple_hbar( tracking, 'Overlaps', 'Percent', title='percent' ) grid = gridplot([tracking_bar, tracking_bar_perc], ncols=2, plot_width=400, plot_height=400) section.plot(grid) def pychopper_plots(report, df): """Make plots from pychopper.cdna_classifier.py. :param report: aplanat WFReport :param df: result DataFrame """ section = report.add_section() section.markdown(''' ### pychopper summary statisitcs The following plots summarize the output of [cdna_classifier.py] (https://github.com/nanoporetech/pychopper) * **Classification of output reads**: * Primers_found: Reads with primers found in correct orientation at both ends. * Rescue: Reads 'rescued' from fused reads * Unusable: Read with missing or incorrect primer orientation * **Strand of oriented reads**: * Strand of read relative to the mRNA * **Strand of rescued read**: * Strand of read that were rescued from fused reads * **Number of primer alignment hits in unclassified reads**: * Note: Need to look into what this means * **Number of primer alignment hits in rescued reads**: * Note: Need to look into what this means * **Number of usable segments per rescued read**: * Number of usable segments (primer-flanked, correctly oriented regions) per fused read. * **Usable bases as a function of cutoff**: * The cutoff value supplied to the primer alignment tool. Note: What are usabel bases in this conext * ** Log10 length distribution of trimmed away sequences**: * todo ''') def g(df, index, title): df_ = df[df.index == index] groups = df_.Name.values bar_ = bars.simple_bar( groups, df_['Value'].values, title=title, colors=Colors.cerulean) return bar_ df1 = df.set_index('Name', drop=True) df1 = df1.T[['Primers_found', 'Rescue', 'Unusable']] bar_class = bars.simple_bar( df1.columns.values, df1.iloc[0].values, title='Classification of output reads', colors=Colors.cerulean) plots = [ bar_class, g(df, 'Strand', 'Strand of oriented read'), g(df, 'RescueStrand', 'Strand of rescued reads'), g(df, 'UnclassHitNr', 'Number of hits in unclassified reads'), g(df, 'RescueHitNr', 'Number of hits in rescued reads'), g(df, 'RescueSegmentNr', 'Number of usable segments per rescued read') ] q = round(df.loc['Parameter', 'Value'], 4) df_at = df[df.index == 'AutotuneSample'].astype('float') # Add vertical line at x=q ymin, ymax = df_at['Value'].min(), df_at['Value'].max() plots.append(lines.line([df_at['Name'].values.tolist(), [q, q]], [df_at['Value'].values.tolist(), [ymin, ymax]], title=("Usable bases as function of cutoff(q).Best" " q={}").format(q), colors=['blue', 'red'] )) df_unusable = df[df.index == 'Unusable'].astype('float') plots.append(lines.line([np.log10(1 + df_unusable['Name'])], [df_unusable['Value']], title=("Log10 length distribution of trimmed away" " sequences.") )) grid = gridplot(plots, ncols=2, plot_width=400, plot_height=400) section.plot(grid) def main(): """Run the entry point.""" parser = argparse.ArgumentParser() parser.add_argument("report", help="Report output file") parser.add_argument("summaries", nargs='+', help="Read summary file.") parser.add_argument( "--versions", required=True, help="directory containing CSVs containing name,version.") parser.add_argument( "--params", default=None, required=True, help="A JSON file containing the workflow parameter key/values") parser.add_argument( "--revision", default='unknown', help="git branch/tag of the executed workflow") parser.add_argument( "--commit", default='unknown', help="git commit of the executed workflow") parser.add_argument( "--alignment_stats", required=True, help="TSV summary file of alignment statistics") parser.add_argument( "--gffcompare_tracking", required=False, default=None, help="TSV summary file of alignment statistics") parser.add_argument( "--gffcompare_stats", required=False, default=None, help="TSV summary file of alignment statistics") parser.add_argument( "--pychop_report", required=True, help="TSV summary file of pychopper statistics") args = parser.parse_args() report = WFReport( "Workflow for assembling transcript isoforms", "wf-isoforms", revision=args.revision, commit=args.commit) # Add reads summary section report.add_section( section=fastcat.full_report(args.summaries)) # workflow-specific plotting workflow_plots(report, args.alignment_stats, args.gffcompare_stats, args.gffcompare_tracking) if args.pychop_report: df_chop_stats = pd.read_csv(args.pychop_report, sep='\t', index_col=0) pychopper_plots(report, df_chop_stats) # Arguments and software versions report.add_section( section=scomponents.version_table(args.versions)) report.add_section( section=scomponents.params_table(args.params)) # write report report.write(args.report) if __name__ == "__main__": main()