Commit 2c7bff08 authored by Julius Metz's avatar Julius Metz

generate html file with plots for one collectlfile

parent 035549a8
......@@ -24,6 +24,7 @@ def hardvalue(filter_data, filtervalue):
int(100 * (cmd_data_value) / filter_data[cmd_data_key]),
cmd,
))
for key, values in tmp_sort_data.items():
current_percent = 0
for (percent, cmd) in sorted(values, key=lambda x: x[0], reverse=True):
......
import plotly.io as pio
import plotly
import datetime
def build_plot(plot_data, title=None, xtitle=None, ytitle=None, showlegend=True, **kwargs):
def build_plot(plot_data, **kwargs):
plot = {
'data': plot_data,
'layout': {
'height': kwargs.get('height', 500),
'width': kwargs.get('width', None),
'title': {
'text': title
'text': kwargs.get('title', None),
},
'xaxis': {
'title': xtitle,
'title': kwargs.get('xtitle', None),
},
'yaxis': {
'title': ytitle,
'title': kwargs.get('ytitle', None),
'type': kwargs.get('ytype', None),
},
'showlegend': showlegend,
'showlegend': kwargs.get('showlegend', True),
},
}
return plot
return plotly.offline.plot(plot, include_plotlyjs=False, output_type='div')
def cpu_plot(cmds_data, filter_info, **kwargs):
needed_key = 'PCT'
def default_plot(cmds_data, filter_info, plot_settings, **kwargs):
plot_data = []
needed_key = None
if 'needed_key' in kwargs:
needed_key = kwargs['needed_key']
else:
raise Exception('no needed_key in default_plot generater settings')
for cmd in filter_info[needed_key]:
plot_data.append(
{
'type': 'scatter',
'type': 'scattergl',
'mode': 'markers',
'x': cmds_data[cmd].get('datetime', []),
'y': cmds_data[cmd].get(needed_key, []),
'name': cmd,
}
)
pio.show(build_plot(plot_data, **kwargs))
return build_plot(plot_data, **kwargs)
def ram_plot(cmds_data, filter_info, **kwargs):
plot_data = []
needed_key = 'VmRSS'
for cmd in filter_info[needed_key]:
cmd_needed_key_data = cmds_data[cmd].get(needed_key, [])
plot_data.append(
{
'type': 'scatter',
'mode': 'markers',
'x': cmds_data[cmd].get('datetime', []),
'y': [ value / 1024 / 1024 for value in cmd_needed_key_data],
'name': cmd,
}
)
pio.show(build_plot(plot_data, **kwargs))
return build_plot(plot_data, **kwargs)
return build_plot(plot_data, **plot_settings)
......@@ -2,12 +2,18 @@ import re
import subprocess
import shlex
from pathlib import Path
import copy
import time
import datetime
import click
import plotly
from yattag import Doc
import plots_generators
import filter_func
import value_merger
FILTER_FUNCTIONS = ['hardvalue', 'average']
......@@ -20,28 +26,74 @@ NAME_SPEZIAL_PARAMETER_CONFIG = {
'python': ['-m', '-W', '-X', '--check-hash-based-pycs', '-c'],
}
MERGE_VALUES = ['PCT', 'VmRSS']
NEEDED_VALUES_DEFAULTS = {
'default_value': 0,
'default_merger': 'addition_int',
}
NEEDED_VALUES = {
'PCT': {'keys':['PCT']},
'VmRSS': {'keys':['VmRSS'], 'merger': 'x2oneaddition_float_sizedown2'},
'RKBC': {'keys':['RKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'WKBC': {'keys':['WKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'syscalls': {'keys':['RSYS', 'WSYS'], 'merger': 'x2oneaddtion_int'},
}
COMAND_BLACKLIST_REGEX = [
r'^[^ ]+perl .+collectl',
]
PLOT_CONFIG = [{
'generator': 'cpu_plot',
'settings': {
'generator': 'default_plot',
'name': 'cpu',
'generator_settings': {'needed_key':'PCT'},
'plotly_settings': {
'title': 'CPU load',
'xtitle': 'Date',
'ytitle': 'CPU load',
},
},
{
'generator': 'ram_plot',
'settings': {
'generator': 'default_plot',
'name': 'ram',
'generator_settings': {'needed_key': 'VmRSS'},
'plotly_settings': {
'title': 'Memory Usage',
'xtitle': 'Date',
'ytitle': 'RAM usage GiB',
},
},
{
'generator': 'default_plot',
'name': 'ior',
'generator_settings': {'needed_key':'RKBC'},
'plotly_settings': {
'title': 'read io',
'xtitle': 'Date',
'ytitle': 'I/O MiB/s',
'ytype': 'log',
},
},
{
'generator': 'default_plot',
'name': 'iow',
'generator_settings': {'needed_key':'WKBC'},
'plotly_settings': {
'title': 'write io',
'xtitle': 'Date',
'ytitle': 'I/O MiB/s',
'ytype': 'log',
},
},
{
'generator': 'default_plot',
'name': 'ios',
'generator_settings': {'needed_key':'syscalls'},
'plotly_settings': {
'title': 'I/O syscalls',
'xtitle': 'Date',
'ytitle': 'I/O syscalls/s',
'ytype': 'log',
},
},
]
......@@ -97,9 +149,12 @@ def get_cmdname(cmd, coarsest=False):
def parse_file(path, collectl, merge, coarsest):
collectl_starttime = time.time()
process = subprocess.run(
[collectl, '-P', '-p', path, '-sZ'], capture_output=True,
)
print('collectl make table took {:.1f}s'.format(time.time()- collectl_starttime))
parsing_starttime = time.time()
output = process.stdout.decode().splitlines()
head = output.pop(0).split(' ')
for possible_head in output[:]:
......@@ -112,6 +167,11 @@ def parse_file(path, collectl, merge, coarsest):
head[0] = head[0][1:]
head_indexes_dict = {head_title: index for index, head_title in enumerate(head)}
entrys_data = {}
empty_dict_with_value_titles = {
value_title: copy.deepcopy(
value_title_settings.get('default', NEEDED_VALUES_DEFAULTS['default_value'])
) for value_title, value_title_settings in NEEDED_VALUES.items()
}
for entry in output:
splited_entry = entry.split(' ', len(head_indexes_dict)-1)
cmd = splited_entry[-1]
......@@ -128,20 +188,27 @@ def parse_file(path, collectl, merge, coarsest):
datetime.time.fromisoformat(splited_entry[head_indexes_dict['Time']]),
)
if not tmp_datetime in entrys_data[cmd]:
entrys_data[cmd][tmp_datetime] = {
key: 0.0 for key in MERGE_VALUES
}
for head_title in MERGE_VALUES:
entrys_data[cmd][tmp_datetime][head_title] += float(splited_entry[head_indexes_dict[head_title]])
entrys_data[cmd][tmp_datetime] = copy.deepcopy(empty_dict_with_value_titles)
for value_title, value_title_settings in NEEDED_VALUES.items():
entrys_data[cmd][tmp_datetime][value_title] = getattr(
value_merger,
value_title_settings.get('merger', NEEDED_VALUES_DEFAULTS['default_merger']),
)(
entrys_data[cmd][tmp_datetime][value_title],
*[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
)
#float(splited_entry[head_indexes_dict[head_title]])
print('parsing/merge took {:.1f}s'.format(time.time()- parsing_starttime))
dictbuild_starttime = time.time()
entry_data_plotfriendly = {}
plot_filter_data = {key: 0.0 for key in MERGE_VALUES}
plot_filter_data = copy.deepcopy(empty_dict_with_value_titles)
plot_filter_data['number_of_values'] = 0
plot_filter_data['commands'] = {}
for cmd, cmd_data in entrys_data.items():
plot_filter_data['commands'][cmd] = {key: 0 for key in MERGE_VALUES}
plot_filter_data['commands'][cmd] = copy.deepcopy(empty_dict_with_value_titles)
plot_filter_data['commands'][cmd]['number_of_values'] = 0
entry_data_plotfriendly[cmd] = {key: [] for key in MERGE_VALUES}
entry_data_plotfriendly[cmd] = {key: [] for key in NEEDED_VALUES.keys()}
entry_data_plotfriendly[cmd]['datetime'] = []
for cmd_data_time, cmd_data_values in cmd_data.items():
entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
......@@ -152,30 +219,89 @@ def parse_file(path, collectl, merge, coarsest):
plot_filter_data['commands'][cmd]['number_of_values'] += 1
plot_filter_data[cmd_data_key] += cmd_data_value
plot_filter_data['number_of_values'] += 1
print('data dict/ filter_data_dict build took {:.1f}s'.format(time.time()- dictbuild_starttime))
return entry_data_plotfriendly, plot_filter_data
def build_html(plots_dict, plotlypath):
doc, tag, text = Doc().tagtext()
doc.asis('<!DOCTYPE html>')
with tag('html'):
with tag('head'):
with tag('script'):
with Path(plotlypath).open(mode='r') as f:
doc.asis(f.read())
with tag('script'):
doc.asis('''toggleplot = (name, index) => {
const elem = document.querySelector('#'+name+' div:nth-child('+index+')');
if (window.getComputedStyle(elem).display === "none") {
elem.style.display = 'contents';
} else {
elem.style.display = 'none';
}
}''')
with tag('body'):
with tag('div', id='index'):
with tag('h2'):
text('Index:')
with tag('ul'):
for name in plots_dict.keys():
with tag('li'):
with tag('a', href='#'+name):
text(name)
for name, plots in plots_dict.items():
with tag('div', id=name):
with tag('h2'):
text(name)
for i, plot in enumerate(plots):
with tag('button', onclick='toggleplot("{}", {})'.format(name, i+1)):
doc.asis('toggle host')
doc.asis(plot)
return doc.getvalue()
@click.command()
@click.option('--file', '-f', required=True)
@click.option('--collectl', '-c', required=False, default='collectl')
@click.option('--plotlypath', '-p', required=True)
@click.option('--destination', '-d', required=False, default='.')
@click.option('--merge/--notmerge', default=True)
@click.option('--coarsest/--notcoarsest', default=False)
@click.option('--filtercmd/--notfiltercmd', default=True)
@click.option('--filtercmds/--notfiltercmds', default=True)
@click.option('--filtervalue', '-v', type=int, default=90)
@click.option('--filtertype', '-t',
type=click.Choice(FILTER_FUNCTIONS, case_sensitive=False),
default=FILTER_FUNCTIONS[0])
def main(file, collectl, merge, coarsest, filtercmd, filtervalue, filtertype):
def main(file, collectl, plotlypath, destination, merge, coarsest, filtercmds, filtervalue, filtertype):
path = Path(file)
if path.exists():
data, filter_data = parse_file(path, collectl, merge, coarsest)
if filtercmd:
if filtercmds:
filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
else:
filter_infos = {key: list(data.keys()) for key in NEEDED_VALUES.keys()}
plots_dict = {config['name']: [] for config in PLOT_CONFIG}
for plot_config in PLOT_CONFIG:
plot = getattr(plots_generators, plot_config['generator'])(data, filter_infos, **plot_config['settings'])
plots_dict[plot_config['name']].append(getattr(plots_generators, plot_config['generator'])(
data,
filter_infos,
plot_config['plotly_settings'],
**plot_config['generator_settings'],
))
with Path(destination, 'plots.html').open(mode='w') as f:
f.write(build_html(plots_dict, plotlypath))
......
def addition_int(old_v, to_merge_v):
"""sum old_v and int(to_merge_v)
Arguments:
old_v {int} -- first value
to_merge_v {str/int/float} -- value to add to old_v
Returns:
int -- sum of old_v and intto_merge_v
"""
return old_v + int(to_merge_v)
def x2oneaddtion_int(old_v, *args):
"""sum x values to old_v
Arguments:
old_v {int} -- first value
Returns:
int -- sum x values to old_v
"""
return old_v + sum([int(x) for x in args])
def x2oneaddition_float_sizedown1(old_v, *args):
"""sum x values / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 1024
"""
return old_v + (sum([float(x) for x in args]) / 1024)
def x2oneaddition_float_sizedown2(old_v, *args):
"""sum x values / 1024 / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 1024 / 1024
"""
return old_v + (sum([float(x) for x in args]) / 1024 / 1024)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment