Commit 2c7bff08 authored by Julius Metz's avatar Julius Metz

generate html file with plots for one collectlfile

parent 035549a8
...@@ -24,6 +24,7 @@ def hardvalue(filter_data, filtervalue): ...@@ -24,6 +24,7 @@ def hardvalue(filter_data, filtervalue):
int(100 * (cmd_data_value) / filter_data[cmd_data_key]), int(100 * (cmd_data_value) / filter_data[cmd_data_key]),
cmd, cmd,
)) ))
for key, values in tmp_sort_data.items(): for key, values in tmp_sort_data.items():
current_percent = 0 current_percent = 0
for (percent, cmd) in sorted(values, key=lambda x: x[0], reverse=True): for (percent, cmd) in sorted(values, key=lambda x: x[0], reverse=True):
......
import plotly.io as pio import plotly
import datetime import datetime
def build_plot(plot_data, title=None, xtitle=None, ytitle=None, showlegend=True, **kwargs): def build_plot(plot_data, **kwargs):
plot = { plot = {
'data': plot_data, 'data': plot_data,
'layout': { 'layout': {
'height': kwargs.get('height', 500),
'width': kwargs.get('width', None),
'title': { 'title': {
'text': title 'text': kwargs.get('title', None),
}, },
'xaxis': { 'xaxis': {
'title': xtitle, 'title': kwargs.get('xtitle', None),
}, },
'yaxis': { 'yaxis': {
'title': ytitle, 'title': kwargs.get('ytitle', None),
'type': kwargs.get('ytype', None),
}, },
'showlegend': showlegend, 'showlegend': kwargs.get('showlegend', True),
}, },
} }
return plot return plotly.offline.plot(plot, include_plotlyjs=False, output_type='div')
def default_plot(cmds_data, filter_info, plot_settings, **kwargs):
def cpu_plot(cmds_data, filter_info, **kwargs):
needed_key = 'PCT'
plot_data = [] plot_data = []
needed_key = None
if 'needed_key' in kwargs:
needed_key = kwargs['needed_key']
else:
raise Exception('no needed_key in default_plot generater settings')
for cmd in filter_info[needed_key]: for cmd in filter_info[needed_key]:
plot_data.append( plot_data.append(
{ {
'type': 'scatter', 'type': 'scattergl',
'mode': 'markers', 'mode': 'markers',
'x': cmds_data[cmd].get('datetime', []), 'x': cmds_data[cmd].get('datetime', []),
'y': cmds_data[cmd].get(needed_key, []), 'y': cmds_data[cmd].get(needed_key, []),
'name': cmd, 'name': cmd,
} }
) )
pio.show(build_plot(plot_data, **kwargs)) return build_plot(plot_data, **plot_settings)
return build_plot(plot_data, **kwargs)
def ram_plot(cmds_data, filter_info, **kwargs):
plot_data = []
needed_key = 'VmRSS'
for cmd in filter_info[needed_key]:
cmd_needed_key_data = cmds_data[cmd].get(needed_key, [])
plot_data.append(
{
'type': 'scatter',
'mode': 'markers',
'x': cmds_data[cmd].get('datetime', []),
'y': [ value / 1024 / 1024 for value in cmd_needed_key_data],
'name': cmd,
}
)
pio.show(build_plot(plot_data, **kwargs))
return build_plot(plot_data, **kwargs)
...@@ -2,12 +2,18 @@ import re ...@@ -2,12 +2,18 @@ import re
import subprocess import subprocess
import shlex import shlex
from pathlib import Path from pathlib import Path
import copy
import time
import datetime import datetime
import click import click
import plotly
from yattag import Doc
import plots_generators import plots_generators
import filter_func import filter_func
import value_merger
FILTER_FUNCTIONS = ['hardvalue', 'average'] FILTER_FUNCTIONS = ['hardvalue', 'average']
...@@ -20,28 +26,74 @@ NAME_SPEZIAL_PARAMETER_CONFIG = { ...@@ -20,28 +26,74 @@ NAME_SPEZIAL_PARAMETER_CONFIG = {
'python': ['-m', '-W', '-X', '--check-hash-based-pycs', '-c'], 'python': ['-m', '-W', '-X', '--check-hash-based-pycs', '-c'],
} }
NEEDED_VALUES_DEFAULTS = {
MERGE_VALUES = ['PCT', 'VmRSS'] 'default_value': 0,
'default_merger': 'addition_int',
}
NEEDED_VALUES = {
'PCT': {'keys':['PCT']},
'VmRSS': {'keys':['VmRSS'], 'merger': 'x2oneaddition_float_sizedown2'},
'RKBC': {'keys':['RKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'WKBC': {'keys':['WKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'syscalls': {'keys':['RSYS', 'WSYS'], 'merger': 'x2oneaddtion_int'},
}
COMAND_BLACKLIST_REGEX = [ COMAND_BLACKLIST_REGEX = [
r'^[^ ]+perl .+collectl', r'^[^ ]+perl .+collectl',
] ]
PLOT_CONFIG = [{ PLOT_CONFIG = [{
'generator': 'cpu_plot', 'generator': 'default_plot',
'settings': { 'name': 'cpu',
'generator_settings': {'needed_key':'PCT'},
'plotly_settings': {
'title': 'CPU load', 'title': 'CPU load',
'xtitle': 'Date', 'xtitle': 'Date',
'ytitle': 'CPU load', 'ytitle': 'CPU load',
}, },
}, },
{ {
'generator': 'ram_plot', 'generator': 'default_plot',
'settings': { 'name': 'ram',
'generator_settings': {'needed_key': 'VmRSS'},
'plotly_settings': {
'title': 'Memory Usage', 'title': 'Memory Usage',
'xtitle': 'Date', 'xtitle': 'Date',
'ytitle': 'RAM usage GiB', 'ytitle': 'RAM usage GiB',
}, },
}, },
{
'generator': 'default_plot',
'name': 'ior',
'generator_settings': {'needed_key':'RKBC'},
'plotly_settings': {
'title': 'read io',
'xtitle': 'Date',
'ytitle': 'I/O MiB/s',
'ytype': 'log',
},
},
{
'generator': 'default_plot',
'name': 'iow',
'generator_settings': {'needed_key':'WKBC'},
'plotly_settings': {
'title': 'write io',
'xtitle': 'Date',
'ytitle': 'I/O MiB/s',
'ytype': 'log',
},
},
{
'generator': 'default_plot',
'name': 'ios',
'generator_settings': {'needed_key':'syscalls'},
'plotly_settings': {
'title': 'I/O syscalls',
'xtitle': 'Date',
'ytitle': 'I/O syscalls/s',
'ytype': 'log',
},
},
] ]
...@@ -97,9 +149,12 @@ def get_cmdname(cmd, coarsest=False): ...@@ -97,9 +149,12 @@ def get_cmdname(cmd, coarsest=False):
def parse_file(path, collectl, merge, coarsest): def parse_file(path, collectl, merge, coarsest):
collectl_starttime = time.time()
process = subprocess.run( process = subprocess.run(
[collectl, '-P', '-p', path, '-sZ'], capture_output=True, [collectl, '-P', '-p', path, '-sZ'], capture_output=True,
) )
print('collectl make table took {:.1f}s'.format(time.time()- collectl_starttime))
parsing_starttime = time.time()
output = process.stdout.decode().splitlines() output = process.stdout.decode().splitlines()
head = output.pop(0).split(' ') head = output.pop(0).split(' ')
for possible_head in output[:]: for possible_head in output[:]:
...@@ -112,6 +167,11 @@ def parse_file(path, collectl, merge, coarsest): ...@@ -112,6 +167,11 @@ def parse_file(path, collectl, merge, coarsest):
head[0] = head[0][1:] head[0] = head[0][1:]
head_indexes_dict = {head_title: index for index, head_title in enumerate(head)} head_indexes_dict = {head_title: index for index, head_title in enumerate(head)}
entrys_data = {} entrys_data = {}
empty_dict_with_value_titles = {
value_title: copy.deepcopy(
value_title_settings.get('default', NEEDED_VALUES_DEFAULTS['default_value'])
) for value_title, value_title_settings in NEEDED_VALUES.items()
}
for entry in output: for entry in output:
splited_entry = entry.split(' ', len(head_indexes_dict)-1) splited_entry = entry.split(' ', len(head_indexes_dict)-1)
cmd = splited_entry[-1] cmd = splited_entry[-1]
...@@ -128,20 +188,27 @@ def parse_file(path, collectl, merge, coarsest): ...@@ -128,20 +188,27 @@ def parse_file(path, collectl, merge, coarsest):
datetime.time.fromisoformat(splited_entry[head_indexes_dict['Time']]), datetime.time.fromisoformat(splited_entry[head_indexes_dict['Time']]),
) )
if not tmp_datetime in entrys_data[cmd]: if not tmp_datetime in entrys_data[cmd]:
entrys_data[cmd][tmp_datetime] = { entrys_data[cmd][tmp_datetime] = copy.deepcopy(empty_dict_with_value_titles)
key: 0.0 for key in MERGE_VALUES for value_title, value_title_settings in NEEDED_VALUES.items():
} entrys_data[cmd][tmp_datetime][value_title] = getattr(
for head_title in MERGE_VALUES: value_merger,
entrys_data[cmd][tmp_datetime][head_title] += float(splited_entry[head_indexes_dict[head_title]]) value_title_settings.get('merger', NEEDED_VALUES_DEFAULTS['default_merger']),
)(
entrys_data[cmd][tmp_datetime][value_title],
*[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
)
#float(splited_entry[head_indexes_dict[head_title]])
print('parsing/merge took {:.1f}s'.format(time.time()- parsing_starttime))
dictbuild_starttime = time.time()
entry_data_plotfriendly = {} entry_data_plotfriendly = {}
plot_filter_data = {key: 0.0 for key in MERGE_VALUES} plot_filter_data = copy.deepcopy(empty_dict_with_value_titles)
plot_filter_data['number_of_values'] = 0 plot_filter_data['number_of_values'] = 0
plot_filter_data['commands'] = {} plot_filter_data['commands'] = {}
for cmd, cmd_data in entrys_data.items(): for cmd, cmd_data in entrys_data.items():
plot_filter_data['commands'][cmd] = {key: 0 for key in MERGE_VALUES} plot_filter_data['commands'][cmd] = copy.deepcopy(empty_dict_with_value_titles)
plot_filter_data['commands'][cmd]['number_of_values'] = 0 plot_filter_data['commands'][cmd]['number_of_values'] = 0
entry_data_plotfriendly[cmd] = {key: [] for key in MERGE_VALUES} entry_data_plotfriendly[cmd] = {key: [] for key in NEEDED_VALUES.keys()}
entry_data_plotfriendly[cmd]['datetime'] = [] entry_data_plotfriendly[cmd]['datetime'] = []
for cmd_data_time, cmd_data_values in cmd_data.items(): for cmd_data_time, cmd_data_values in cmd_data.items():
entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time) entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
...@@ -152,30 +219,89 @@ def parse_file(path, collectl, merge, coarsest): ...@@ -152,30 +219,89 @@ def parse_file(path, collectl, merge, coarsest):
plot_filter_data['commands'][cmd]['number_of_values'] += 1 plot_filter_data['commands'][cmd]['number_of_values'] += 1
plot_filter_data[cmd_data_key] += cmd_data_value plot_filter_data[cmd_data_key] += cmd_data_value
plot_filter_data['number_of_values'] += 1 plot_filter_data['number_of_values'] += 1
print('data dict/ filter_data_dict build took {:.1f}s'.format(time.time()- dictbuild_starttime))
return entry_data_plotfriendly, plot_filter_data return entry_data_plotfriendly, plot_filter_data
def build_html(plots_dict, plotlypath):
doc, tag, text = Doc().tagtext()
doc.asis('<!DOCTYPE html>')
with tag('html'):
with tag('head'):
with tag('script'):
with Path(plotlypath).open(mode='r') as f:
doc.asis(f.read())
with tag('script'):
doc.asis('''toggleplot = (name, index) => {
const elem = document.querySelector('#'+name+' div:nth-child('+index+')');
if (window.getComputedStyle(elem).display === "none") {
elem.style.display = 'contents';
} else {
elem.style.display = 'none';
}
}''')
with tag('body'):
with tag('div', id='index'):
with tag('h2'):
text('Index:')
with tag('ul'):
for name in plots_dict.keys():
with tag('li'):
with tag('a', href='#'+name):
text(name)
for name, plots in plots_dict.items():
with tag('div', id=name):
with tag('h2'):
text(name)
for i, plot in enumerate(plots):
with tag('button', onclick='toggleplot("{}", {})'.format(name, i+1)):
doc.asis('toggle host')
doc.asis(plot)
return doc.getvalue()
@click.command() @click.command()
@click.option('--file', '-f', required=True) @click.option('--file', '-f', required=True)
@click.option('--collectl', '-c', required=False, default='collectl') @click.option('--collectl', '-c', required=False, default='collectl')
@click.option('--plotlypath', '-p', required=True)
@click.option('--destination', '-d', required=False, default='.')
@click.option('--merge/--notmerge', default=True) @click.option('--merge/--notmerge', default=True)
@click.option('--coarsest/--notcoarsest', default=False) @click.option('--coarsest/--notcoarsest', default=False)
@click.option('--filtercmd/--notfiltercmd', default=True) @click.option('--filtercmds/--notfiltercmds', default=True)
@click.option('--filtervalue', '-v', type=int, default=90) @click.option('--filtervalue', '-v', type=int, default=90)
@click.option('--filtertype', '-t', @click.option('--filtertype', '-t',
type=click.Choice(FILTER_FUNCTIONS, case_sensitive=False), type=click.Choice(FILTER_FUNCTIONS, case_sensitive=False),
default=FILTER_FUNCTIONS[0]) default=FILTER_FUNCTIONS[0])
def main(file, collectl, merge, coarsest, filtercmd, filtervalue, filtertype): def main(file, collectl, plotlypath, destination, merge, coarsest, filtercmds, filtervalue, filtertype):
path = Path(file) path = Path(file)
if path.exists(): if path.exists():
data, filter_data = parse_file(path, collectl, merge, coarsest) data, filter_data = parse_file(path, collectl, merge, coarsest)
if filtercmd: if filtercmds:
filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue) filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
else:
filter_infos = {key: list(data.keys()) for key in NEEDED_VALUES.keys()}
plots_dict = {config['name']: [] for config in PLOT_CONFIG}
for plot_config in PLOT_CONFIG: for plot_config in PLOT_CONFIG:
plot = getattr(plots_generators, plot_config['generator'])(data, filter_infos, **plot_config['settings']) plots_dict[plot_config['name']].append(getattr(plots_generators, plot_config['generator'])(
data,
filter_infos,
plot_config['plotly_settings'],
**plot_config['generator_settings'],
))
with Path(destination, 'plots.html').open(mode='w') as f:
f.write(build_html(plots_dict, plotlypath))
......
def addition_int(old_v, to_merge_v):
"""sum old_v and int(to_merge_v)
Arguments:
old_v {int} -- first value
to_merge_v {str/int/float} -- value to add to old_v
Returns:
int -- sum of old_v and intto_merge_v
"""
return old_v + int(to_merge_v)
def x2oneaddtion_int(old_v, *args):
"""sum x values to old_v
Arguments:
old_v {int} -- first value
Returns:
int -- sum x values to old_v
"""
return old_v + sum([int(x) for x in args])
def x2oneaddition_float_sizedown1(old_v, *args):
"""sum x values / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 1024
"""
return old_v + (sum([float(x) for x in args]) / 1024)
def x2oneaddition_float_sizedown2(old_v, *args):
"""sum x values / 1024 / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 1024 / 1024
"""
return old_v + (sum([float(x) for x in args]) / 1024 / 1024)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment