Commit 53da944e authored by Matthias Lieber's avatar Matthias Lieber

changes after code review

parent ed2673ee
......@@ -49,13 +49,16 @@ def get_cmdname(cmd, coarsest=False):
Returns:
str -- new cmd name
"""
# search for (for example) bash_function="python" in cmd="/usr/bin/python3 my_script.py"
cmd_splited = shlex.split(cmd)
bash_function = cmd_splited[0].split('/')[-1]
bash_function = re.search(r'[^\W\n]+', bash_function).group(0)
# check if bash_function is known, if not, return bash_function
spezial_parameter = NAME_SPEZIAL_PARAMETER_CONFIG.get(bash_function, None)
if coarsest or spezial_parameter is None:
return bash_function
skip = False
# search script/program name within the parameters and only return this without path
for position, parameter in enumerate(cmd_splited[1:]):
if skip:
skip = False
......@@ -86,13 +89,16 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
(dict, dict) -- 1.: parsed_data 2.: data for filter function
"""
collectl_starttime = time.time()
# run collectl in playback mode and read output into list
process = subprocess.run(
[collectl, '-P', '-p', path, '-sZ'], capture_output=True, check=True,
[collectl, '-P', '-p', path, '-sZ'], stdout=subprocess.PIPE, check=True,
)
print('collectl make table took {:.1f}s'.format(
time.time() - collectl_starttime))
parsing_starttime = time.time()
# output contains all data!
output = process.stdout.decode().splitlines()
# get table head
head = output.pop(0).split(' ')
for possible_head in output[:]:
if possible_head.startswith('#'):
......@@ -101,10 +107,10 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
else:
break
# get template of an entry from the head
head[0] = head[0][1:]
head_indexes_dict = {
head_title: index for index, head_title in enumerate(head)}
entrys_data = {}
empty_dict_with_value_titles = {
value_title: copy.deepcopy(
value_title_settings.get(
......@@ -112,9 +118,14 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
)
) for value_title, value_title_settings in NEEDED_VALUES.items()
}
# parse all output lines
entrys_data = {}
cmd_cmdshort_dict = {}
for entry in output:
# split by ' ' (exclude command from splitting)
splited_entry = entry.split(' ', len(head_indexes_dict)-1)
# get command string and shorten
cmd = splited_entry[-1]
if shorten_cmds or coarsest:
if cmd in cmd_cmdshort_dict:
......@@ -125,17 +136,20 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
cmd = short_cmd
if cmd in COMAND_BLACKLIST:
continue
# create dict for each command
if not cmd in entrys_data:
entrys_data[cmd] = {}
# get datetime obj for current entry
tmp_datetime = datetime.datetime.combine(
datestr2date(splited_entry[head_indexes_dict['Date']]),
datetime.time.fromisoformat(
splited_entry[head_indexes_dict['Time']],
),
datetime.time(*[int(n) for n in splited_entry[head_indexes_dict['Time']].split(':')]),
)
# if datetime not yet existing, add new entry from template
if not tmp_datetime in entrys_data[cmd]:
entrys_data[cmd][tmp_datetime] = copy.deepcopy(
empty_dict_with_value_titles)
# get values from data as given in NEEDED_VALUES and run specified merger function
# to merge multiple values with same timestamp or rescale (e.g. to GB)
for value_title, value_title_settings in NEEDED_VALUES.items():
entrys_data[cmd][tmp_datetime][value_title] = getattr(
value_merger,
......@@ -147,6 +161,8 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
print('parsing/merge took {:.1f}s'.format(time.time() - parsing_starttime))
dictbuild_starttime = time.time()
# create lists entry_data_plotfriendly[cmd][metric] = [ values, ... ] with the actual data to plot
# and sum up all metrics for each command to enable filtering of non-interesting commands later on
entry_data_plotfriendly = {}
plot_filter_data = copy.deepcopy(empty_dict_with_value_titles)
plot_filter_data['number_of_values'] = 0
......@@ -260,12 +276,12 @@ def build_html(plots_dict):
text(name)
for name, plots in plots_dict.items():
with tag('div', id=name):
with tag('h2'):
text(name)
#with tag('h2'):
# text(name)
for i, plot in enumerate(plots):
with tag('p'):
with tag('button', onclick='toggleplot("{}", {})'.format(name, i)):
text('toggle plot')
#with tag('p'):
# with tag('button', onclick='toggleplot("{}", {})'.format(name, i)):
# text('toggle plot')
with tag('div', klass=name):
doc.asis(plot)
......@@ -305,7 +321,7 @@ def data_from_file(arguments):
coarsest {bool} -- para for parse_file
filtercmds {bool} -- if True cmds will be filter else not
filtervalue {int} -- para for filter
filtertype {str} -- wich filter will be called
filtertype {str} -- which filter will be called
Returns:
[type] -- [description]
"""
......@@ -331,7 +347,7 @@ def data_from_file(arguments):
@click.option('--shorten/--notshorten', default=True, help='commands will be shorted only to name')
@click.option('--coarsest/--notcoarsest', default=False, help='commands will be shorted only to type (bash, perl, ...)')
@click.option('--filtercmds/--notfiltercmds', default=True, help='filtering or not')
@click.option('--filtervalue', help='Value which is given to the filter.')
@click.option('--filtervalue', help='Parameter which is given to the filter.')
@click.option('--filtertype',
type=click.Choice(FILTER_FUNCTIONS.keys(), case_sensitive=False),
default=DEFAULT_FILTER, show_default=True,
......@@ -366,6 +382,7 @@ def main(source, collectl, plotlypath, destination,
FILTER_FUNCTIONS[filtertype],
))
# use multiprocessing to parse all collectl output files independently from each other in parallel
pool = mp.Pool(min(len(data_colllect_functions), mp.cpu_count()))
results = pool.map(data_from_file, data_colllect_functions)
pool.close()
......@@ -377,12 +394,13 @@ def main(source, collectl, plotlypath, destination,
cmd_colors = {cmd: PLOTLY_COLORS[i % len(PLOTLY_COLORS)] for i, cmd in enumerate(set(cmd_all))}
# for each host and each plot (as given in config) call make_plot to create html div with plotly
start_plots_build = time.time()
plots_dict = {}
for host, host_data in hosts_data.items():
plots_dict[host] = {config['name']: [] for config in PLOT_CONFIG}
for plot_config in PLOT_CONFIG:
plots_dict[host][plot_config['name']].append(make_plot(
plots_dict[host][plot_config['name']].append( make_plot(
host_data['data'],
host_data['filter_infos'],
cmd_colors,
......@@ -393,12 +411,14 @@ def main(source, collectl, plotlypath, destination,
print("plots build in {:.1f}s".format(time.time() - start_plots_build))
# create output directory and copy plotly.js
plots_dir.mkdir()
try:
shutil.copy(plotlypath, str(Path(plots_dir, 'plotly.js')))
except shutil.SameFileError:
pass
# for each host create html file with plots
time_sites = time.time()
for host, site_plots in plots_dict.items():
with Path(plots_dir, host+'-plots.html').open(mode='w') as plots_file:
......
......@@ -16,7 +16,7 @@ NEEDED_VALUES_DEFAULTS = {
# 'merger' : function that merge the values from the given keys to the new for the plots
# 'default': startvalue for merging
NEEDED_VALUES = {
'PCT': {'keys':['PCT']},
'PCT': {'keys':['PCT'], 'merger': 'x2oneaddition_float_by_100'},
'VmRSS': {'keys':['VmRSS'], 'merger': 'x2oneaddition_float_sizedown2'},
'RKBC': {'keys':['RKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'WKBC': {'keys':['WKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
......@@ -106,7 +106,7 @@ barchart_buttons = {
}
PLOT_CONFIG = [{
'name': 'cpu',
'name': 'CPU load',
'needed_key': 'PCT',
'plotly_settings': {
'data': {
......@@ -123,7 +123,7 @@ PLOT_CONFIG = [{
},
},
{
'name': 'ram',
'name': 'RAM usage',
'needed_key': 'VmRSS',
'plotly_settings': {
'data': {
......@@ -140,7 +140,7 @@ PLOT_CONFIG = [{
},
},
{
'name': 'ior',
'name': 'I/O read',
'needed_key': 'RKBC',
'plotly_settings': {
'data': {
......@@ -149,7 +149,7 @@ PLOT_CONFIG = [{
},
'layout': {
'height': 500,
'title': 'read io {host}',
'title': 'I/O read {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'I/O MiB/s', 'type': 'log'},
**barchart_buttons
......@@ -157,7 +157,7 @@ PLOT_CONFIG = [{
},
},
{
'name': 'iow',
'name': 'I/O write',
'needed_key': 'WKBC',
'plotly_settings': {
'data': {
......@@ -166,7 +166,7 @@ PLOT_CONFIG = [{
},
'layout': {
'height': 500,
'title': 'write io {host}',
'title': 'I/O write {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'I/O MiB/s', 'type': 'log'},
**barchart_buttons
......@@ -174,7 +174,7 @@ PLOT_CONFIG = [{
},
},
{
'name': 'ios',
'name': 'I/O syscalls',
'needed_key': 'syscalls',
'plotly_settings': {
'data': {
......
......@@ -47,3 +47,15 @@ def x2oneaddition_float_sizedown2(old_v, *args):
int -- sum x values to old_v / 1024 / 1024
"""
return old_v + (sum([float(x) for x in args]) / 1024 / 1024)
def x2oneaddition_float_by_100(old_v, *args):
"""sum x values / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 100
"""
return old_v + (sum([float(x) for x in args]) / 100)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment