generate html file with plots for one collectlfile

filter_func.py

@@ -24,6 +24,7 @@ def hardvalue(filter_data, filtervalue):
                 int(100 * (cmd_data_value) / filter_data[cmd_data_key]),
                 cmd,
             ))
+
     for key, values in tmp_sort_data.items():
         current_percent = 0
         for (percent, cmd) in sorted(values, key=lambda x: x[0], reverse=True):

plots_generators.py

-import plotly.io as pio
+import plotly
 import datetime
 
 
-def build_plot(plot_data, title=None, xtitle=None, ytitle=None, showlegend=True, **kwargs):
+def build_plot(plot_data, **kwargs):
     plot = {
         'data': plot_data,
         'layout': {
+            'height': kwargs.get('height', 500),
+            'width': kwargs.get('width', None),
             'title': {
-                'text': title
+                'text':  kwargs.get('title', None),
             },
             'xaxis': {
-                'title': xtitle,
+                'title': kwargs.get('xtitle', None),
             },
             'yaxis': {
-                'title': ytitle,
+                'title': kwargs.get('ytitle', None),
+                'type': kwargs.get('ytype', None),
             },
-            'showlegend': showlegend,
+            'showlegend': kwargs.get('showlegend', True),
         },
     }
-    return plot
+    return plotly.offline.plot(plot, include_plotlyjs=False, output_type='div')
 
 
-
-
-def cpu_plot(cmds_data, filter_info, **kwargs):
-    needed_key = 'PCT'
+def default_plot(cmds_data, filter_info, plot_settings, **kwargs):
     plot_data = []
+    needed_key = None
+    if 'needed_key' in kwargs:
+        needed_key = kwargs['needed_key']
+    else:
+        raise Exception('no needed_key in default_plot generater settings')
     for cmd in filter_info[needed_key]:
         plot_data.append(
             {
-            'type': 'scatter',
+            'type': 'scattergl',
             'mode': 'markers',
             'x': cmds_data[cmd].get('datetime', []),
             'y': cmds_data[cmd].get(needed_key, []),
             'name': cmd,
             }
         )
-    pio.show(build_plot(plot_data, **kwargs))
-    return build_plot(plot_data, **kwargs)
-
-
-def ram_plot(cmds_data, filter_info, **kwargs):
-    plot_data = []
-    needed_key = 'VmRSS'
-    for cmd in filter_info[needed_key]:
-        cmd_needed_key_data = cmds_data[cmd].get(needed_key, [])
-        plot_data.append(
-            {
-            'type': 'scatter',
-            'mode': 'markers',
-            'x': cmds_data[cmd].get('datetime', []),
-            'y': [ value / 1024 / 1024 for value in cmd_needed_key_data],
-            'name': cmd,
-            }
-        )
-    pio.show(build_plot(plot_data, **kwargs))
-    return build_plot(plot_data, **kwargs)
-
+    return build_plot(plot_data, **plot_settings)

site_generator.py

@@ -2,12 +2,18 @@ import re
 import subprocess
 import shlex
 from pathlib import Path
+import copy
+import time
 import datetime
 
 import click
+import plotly
+from yattag import Doc
+
 
 import plots_generators
 import filter_func
+import value_merger
 
 
 FILTER_FUNCTIONS = ['hardvalue', 'average']
@@ -20,28 +26,74 @@ NAME_SPEZIAL_PARAMETER_CONFIG = {
     'python': ['-m', '-W', '-X', '--check-hash-based-pycs', '-c'],
 }
 
-
-MERGE_VALUES = ['PCT', 'VmRSS']
+NEEDED_VALUES_DEFAULTS = {
+    'default_value': 0,
+    'default_merger': 'addition_int',
+}
+NEEDED_VALUES = {
+    'PCT': {'keys':['PCT']},
+    'VmRSS': {'keys':['VmRSS'], 'merger': 'x2oneaddition_float_sizedown2'},
+    'RKBC': {'keys':['RKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
+    'WKBC': {'keys':['WKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
+    'syscalls': {'keys':['RSYS', 'WSYS'], 'merger': 'x2oneaddtion_int'},
+}
 
 COMAND_BLACKLIST_REGEX = [
     r'^[^ ]+perl .+collectl',
 ]
 PLOT_CONFIG = [{
-        'generator': 'cpu_plot',
-        'settings': {
+        'generator': 'default_plot',
+        'name': 'cpu',
+        'generator_settings': {'needed_key':'PCT'},
+        'plotly_settings': {
             'title': 'CPU load',
             'xtitle': 'Date',
             'ytitle': 'CPU load',
         },
     },
     {
-        'generator': 'ram_plot',
-        'settings': {
+        'generator': 'default_plot',
+        'name': 'ram',
+        'generator_settings': {'needed_key': 'VmRSS'},
+        'plotly_settings': {
             'title': 'Memory Usage',
             'xtitle': 'Date',
             'ytitle': 'RAM usage GiB',
         },
     },
+    {
+        'generator': 'default_plot',
+        'name': 'ior',
+        'generator_settings': {'needed_key':'RKBC'},
+        'plotly_settings': {
+            'title': 'read io',
+            'xtitle': 'Date',
+            'ytitle': 'I/O MiB/s',
+            'ytype': 'log',
+        },
+    },
+    {
+        'generator': 'default_plot',
+        'name': 'iow',
+        'generator_settings': {'needed_key':'WKBC'},
+        'plotly_settings': {
+            'title': 'write io',
+            'xtitle': 'Date',
+            'ytitle': 'I/O MiB/s',
+            'ytype': 'log',
+        },
+    },
+    {
+        'generator': 'default_plot',
+        'name': 'ios',
+        'generator_settings': {'needed_key':'syscalls'},
+        'plotly_settings': {
+            'title': 'I/O syscalls',
+            'xtitle': 'Date',
+            'ytitle': 'I/O syscalls/s',
+            'ytype': 'log',
+        },
+    },
 ]
 
 
@@ -97,9 +149,12 @@ def get_cmdname(cmd, coarsest=False):
 
 
 def parse_file(path, collectl, merge, coarsest):
+    collectl_starttime = time.time()
     process = subprocess.run(
         [collectl, '-P', '-p', path, '-sZ'], capture_output=True,
     )
+    print('collectl make table took {:.1f}s'.format(time.time()- collectl_starttime))
+    parsing_starttime = time.time()
     output = process.stdout.decode().splitlines()
     head = output.pop(0).split(' ')
     for possible_head in output[:]:
@@ -112,6 +167,11 @@ def parse_file(path, collectl, merge, coarsest):
     head[0] = head[0][1:]
     head_indexes_dict = {head_title: index for index, head_title in enumerate(head)}
     entrys_data = {}
+    empty_dict_with_value_titles = {
+        value_title: copy.deepcopy(
+            value_title_settings.get('default', NEEDED_VALUES_DEFAULTS['default_value'])
+        ) for value_title, value_title_settings in NEEDED_VALUES.items()
+    }
     for entry in output:
         splited_entry = entry.split(' ', len(head_indexes_dict)-1)
         cmd = splited_entry[-1]
@@ -128,20 +188,27 @@ def parse_file(path, collectl, merge, coarsest):
                 datetime.time.fromisoformat(splited_entry[head_indexes_dict['Time']]),
             )
             if not tmp_datetime in entrys_data[cmd]:
-                entrys_data[cmd][tmp_datetime] = {
-                    key: 0.0 for key in MERGE_VALUES
-                }
-            for head_title in MERGE_VALUES:
-                entrys_data[cmd][tmp_datetime][head_title] += float(splited_entry[head_indexes_dict[head_title]])
+                entrys_data[cmd][tmp_datetime] = copy.deepcopy(empty_dict_with_value_titles)
+            for value_title, value_title_settings in NEEDED_VALUES.items():
+                entrys_data[cmd][tmp_datetime][value_title] = getattr(
+                    value_merger,
+                    value_title_settings.get('merger', NEEDED_VALUES_DEFAULTS['default_merger']),
+                )(
+                    entrys_data[cmd][tmp_datetime][value_title],
+                    *[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
+                )
+                #float(splited_entry[head_indexes_dict[head_title]])
+    print('parsing/merge took {:.1f}s'.format(time.time()- parsing_starttime))
+    dictbuild_starttime = time.time()
 
     entry_data_plotfriendly = {}
-    plot_filter_data = {key: 0.0 for key in MERGE_VALUES}
+    plot_filter_data = copy.deepcopy(empty_dict_with_value_titles)
     plot_filter_data['number_of_values'] = 0
     plot_filter_data['commands'] = {}
     for cmd, cmd_data in entrys_data.items():
-        plot_filter_data['commands'][cmd] = {key: 0 for key in MERGE_VALUES}
+        plot_filter_data['commands'][cmd] = copy.deepcopy(empty_dict_with_value_titles)
         plot_filter_data['commands'][cmd]['number_of_values'] = 0
-        entry_data_plotfriendly[cmd] = {key: [] for key in MERGE_VALUES}
+        entry_data_plotfriendly[cmd] = {key: [] for key in NEEDED_VALUES.keys()}
         entry_data_plotfriendly[cmd]['datetime'] = []
         for cmd_data_time,  cmd_data_values in cmd_data.items():
             entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
@@ -152,30 +219,89 @@ def parse_file(path, collectl, merge, coarsest):
                 plot_filter_data['commands'][cmd]['number_of_values'] += 1
                 plot_filter_data[cmd_data_key] += cmd_data_value
                 plot_filter_data['number_of_values'] += 1
-
+    print('data dict/ filter_data_dict build took {:.1f}s'.format(time.time()- dictbuild_starttime))
     return entry_data_plotfriendly, plot_filter_data
 
 
+def build_html(plots_dict, plotlypath):
+    doc, tag, text = Doc().tagtext()
+    doc.asis('<!DOCTYPE html>')
+    with tag('html'):
+        with tag('head'):
+            with tag('script'):
+                with Path(plotlypath).open(mode='r') as f:
+                    doc.asis(f.read())
+            with tag('script'):
+                doc.asis('''toggleplot = (name, index) => {
+                    const elem = document.querySelector('#'+name+' div:nth-child('+index+')');
+                    if (window.getComputedStyle(elem).display === "none") {
+                        elem.style.display = 'contents';
+                    } else {
+                        elem.style.display = 'none';
+                    }
+                }''')
+        with tag('body'):
+            with tag('div', id='index'):
+                with tag('h2'):
+                    text('Index:')
+                with tag('ul'):
+                    for name in plots_dict.keys():
+                        with tag('li'):
+                            with tag('a', href='#'+name):
+                                text(name)
+            for name, plots in plots_dict.items():
+                with tag('div', id=name):
+                    with tag('h2'):
+                        text(name)
+                    for i, plot in enumerate(plots):
+                        with tag('button', onclick='toggleplot("{}", {})'.format(name, i+1)):
+                            doc.asis('toggle host')
+                        doc.asis(plot)
+
+    return doc.getvalue()
+
+
+
 @click.command()
 @click.option('--file', '-f', required=True)
 @click.option('--collectl', '-c', required=False, default='collectl')
+@click.option('--plotlypath', '-p', required=True)
+@click.option('--destination', '-d', required=False, default='.')
 @click.option('--merge/--notmerge', default=True)
 @click.option('--coarsest/--notcoarsest', default=False)
-@click.option('--filtercmd/--notfiltercmd', default=True)
+@click.option('--filtercmds/--notfiltercmds', default=True)
 @click.option('--filtervalue', '-v', type=int, default=90)
 @click.option('--filtertype', '-t',
      type=click.Choice(FILTER_FUNCTIONS, case_sensitive=False),
      default=FILTER_FUNCTIONS[0])
-def main(file, collectl, merge, coarsest, filtercmd, filtervalue, filtertype):
+def main(file, collectl, plotlypath, destination, merge, coarsest, filtercmds, filtervalue, filtertype):
     path = Path(file)
     if path.exists():
         data, filter_data = parse_file(path, collectl, merge, coarsest)
 
-        if filtercmd:
+        if filtercmds:
             filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
+        else:
+            filter_infos = {key: list(data.keys()) for key in NEEDED_VALUES.keys()}
 
+        plots_dict = {config['name']: [] for config in PLOT_CONFIG}
         for plot_config in PLOT_CONFIG:
-            plot = getattr(plots_generators, plot_config['generator'])(data, filter_infos, **plot_config['settings'])
+            plots_dict[plot_config['name']].append(getattr(plots_generators, plot_config['generator'])(
+            data,
+            filter_infos,
+            plot_config['plotly_settings'],
+            **plot_config['generator_settings'],
+            ))
+
+        with Path(destination, 'plots.html').open(mode='w') as f:
+            f.write(build_html(plots_dict, plotlypath))
+
+
+
+
+
+
+
 
 
 

value_merger.py

+
+
+def addition_int(old_v, to_merge_v):
+    """sum old_v and int(to_merge_v)
+
+    Arguments:
+        old_v {int} -- first value
+        to_merge_v {str/int/float} -- value to add to old_v
+
+    Returns:
+        int -- sum of old_v and intto_merge_v
+    """
+    return old_v + int(to_merge_v)
+
+
+def x2oneaddtion_int(old_v, *args):
+    """sum x values to old_v
+
+    Arguments:
+        old_v {int} -- first value
+
+    Returns:
+        int -- sum x values to old_v
+    """
+    return old_v + sum([int(x) for x in args])
+
+
+def x2oneaddition_float_sizedown1(old_v, *args):
+    """sum x values / 1024 to old_v
+
+    Arguments:
+        old_v {float} -- first value
+
+    Returns:
+        int -- sum x values to old_v / 1024
+    """
+    return old_v + (sum([float(x) for x in args]) / 1024)
+
+
+def x2oneaddition_float_sizedown2(old_v, *args):
+    """sum x values / 1024 / 1024 to old_v
+
+    Arguments:
+        old_v {float} -- first value
+
+    Returns:
+        int -- sum x values to old_v / 1024 / 1024
+    """
+    return old_v + (sum([float(x) for x in args]) / 1024 / 1024)