Commit ed2673ee authored by Julius Metz's avatar Julius Metz

WC

parent b5123080
......@@ -17,6 +17,8 @@ from yattag import Doc
import filter_func
import value_merger
from Collectl2plotly_Config import *
from Collectl2plotly_Plots_Config import *
def datestr2date(datestr):
......@@ -110,12 +112,18 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
)
) for value_title, value_title_settings in NEEDED_VALUES.items()
}
cmd_cmdshort_dict = {}
for entry in output:
splited_entry = entry.split(' ', len(head_indexes_dict)-1)
cmd = splited_entry[-1]
if shorten_cmds or coarsest:
cmd = get_cmdname(cmd, coarsest=coarsest)
if cmd in COMAND_BLACKLIST_REGEX:
if cmd in cmd_cmdshort_dict:
cmd = cmd_cmdshort_dict[cmd]
else:
short_cmd = get_cmdname(cmd, coarsest=coarsest)
cmd_cmdshort_dict[cmd]= short_cmd
cmd = short_cmd
if cmd in COMAND_BLACKLIST:
continue
if not cmd in entrys_data:
entrys_data[cmd] = {}
......@@ -166,48 +174,16 @@ def parse_file(path, collectl, shorten_cmds, coarsest):
return entry_data_plotfriendly, plot_filter_data
def build_plot(plot_data, plotly_format_vars, **kwargs):
"""Make plotlayout with values from kwargs and plotly_format_vars.
Build plot as html div.
Arguments:
plot_data {dict} -- plot data in plotly layout
plotly_format_vars {dict} -- values for layout config
Returns:
str -- plotly html div
"""
plot = {
'data': plot_data,
'layout': {
'height': kwargs.get('height', 500),
'width': kwargs.get('width', None),
'title': {
'text': kwargs.get('title', None).format(**plotly_format_vars),
},
'xaxis': {
'title': kwargs.get('xtitle', None).format(**plotly_format_vars),
},
'yaxis': {
'title': kwargs.get('ytitle', None).format(**plotly_format_vars),
'type': kwargs.get('ytype', None),
},
'showlegend': kwargs.get('showlegend', True),
},
}
return plotly.offline.plot(plot, include_plotlyjs=False, output_type='div')
def scatter_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key):
"""Build a list of dicts in Plotlyconf style for scatter plot with the data from cmds_data of needed_key
than call build_plot to generate html plot code.
def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key):
"""Build a list of dicts in Plotlyconf style for the diffrent traces with the data from cmds_data of needed_key
than make plot dict in Plotly style and change '{host}' in titels. After that build plot as html div via Plotly.
Arguments:
cmds_data {dict} -- is the data for the plot example: {'Command': {'datetime':[...], 'cpu': [...]}}
filter_info {dict} -- has the information which cmds will be shown example: {'cpu': [cmds]}
cmd_color {dict} -- assign a fixed plot color to each cmd example: {'Command': 'rgb(0, 0, 128)'}
plot_settings {dict} -- settings for build_plot
plotly_format_vars {dict} -- values for build_plot
plot_settings {dict} -- settings for Plotly
plotly_format_vars {dict} -- values for Plotly settings
needed_key {str} -- key of cmds_data for the values to be use
Returns:
......@@ -215,17 +191,38 @@ def scatter_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format
"""
plot_data = []
for cmd in filter_info[needed_key]:
plot_data.append({
'type': 'scattergl',
'mode': 'markers',
trace = {
'x': cmds_data[cmd].get('datetime', []),
'y': cmds_data[cmd].get(needed_key, []),
'name': cmd,
'marker': {
'color': cmd_color[cmd],
},
})
return build_plot(plot_data, plotly_format_vars, **plot_settings)
}
trace.update(
plot_settings['data'],
)
plot_data.append(trace)
plot = {
'data': plot_data,
'layout': copy.deepcopy(plot_settings['layout']),
}
if 'title' in plot['layout']:
plot['layout']['title'] = plot['layout']['title'].format(**plotly_format_vars)
if 'yaxis' in plot['layout'] and 'title' in plot['layout']['yaxis']:
plot['layout']['yaxis']['title'] = plot['layout']['yaxis']['title'].format(
**plotly_format_vars,
)
if 'xaxis' in plot['layout'] and 'title' in plot['layout']['xaxis']:
plot['layout']['xaxis']['title'] = plot['layout']['xaxis']['title'].format(
**plotly_format_vars,
)
return plotly.offline.plot(plot, include_plotlyjs=False, output_type='div')
def build_html(plots_dict):
......@@ -354,7 +351,7 @@ def main(source, collectl, plotlypath, destination,
plots_dir = Path(destination, 'collectlplots')
if plots_dir.is_dir():
print('in destination "collectldir" already exist')
print('in destination "collectlplots" already exist')
exit(1)
......@@ -385,7 +382,7 @@ def main(source, collectl, plotlypath, destination,
for host, host_data in hosts_data.items():
plots_dict[host] = {config['name']: [] for config in PLOT_CONFIG}
for plot_config in PLOT_CONFIG:
plots_dict[host][plot_config['name']].append(scatter_plot(
plots_dict[host][plot_config['name']].append(make_plot(
host_data['data'],
host_data['filter_infos'],
cmd_colors,
......
......@@ -2,112 +2,6 @@
"""
### Merge configs
# hint: merger function must be in value_merger.py
# Defaults merge infos
# default_value is the startvalue for merging if no is given in NEEDED_VALUES
# default_merger is the merger function that is called if no is given in NEEDED_VALUES
NEEDED_VALUES_DEFAULTS = {
'default_value': 0,
'default_merger': 'addition_int',
}
# Information which values needed for plots
# key : name of value needed for plotconfig
# value: is dict with following configuration options
# 'keys' : must be a list with the table heads from collectl that is to be merge
# 'merger' : function that merge the values from the given keys to the new for the plots
# 'default': startvalue for merging
NEEDED_VALUES = {
'PCT': {'keys':['PCT']},
'VmRSS': {'keys':['VmRSS'], 'merger': 'x2oneaddition_float_sizedown2'},
'RKBC': {'keys':['RKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'WKBC': {'keys':['WKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'syscalls': {'keys':['RSYS', 'WSYS'], 'merger': 'x2oneaddtion_int'},
}
### Plots Config
# List of diagrams to be created
# config of Plot
# 'name': Is the name of the Plot will displayed on the website
# 'needed_key': is a key defined in NEEDED_VALUES for the values that are to be used in der Plot as y
# 'plotly_settings': are plotly layout configs, all not required
# hint {host} will be replaced with real hostname in title, xtitle, ytitle
# title: that will be displayed over plot
# xtitle: that will be displayed on the x axis plot
# ytitle: that will be displayed on the y axis plot
# ytype: what type y axis is example: 'log' for more see Plotly doc
# showlegend: legend displayed or not
# 'height': height of plot
# 'width': width of plot
PLOT_CONFIG = [{
'name': 'cpu',
'needed_key': 'PCT',
'plotly_settings': {
'title': 'CPU load {host}',
'xtitle': 'Date',
'ytitle': 'CPU load',
},
},
{
'name': 'ram',
'needed_key': 'VmRSS',
'plotly_settings': {
'title': 'Memory Usage {host}',
'xtitle': 'Date',
'ytitle': 'RAM usage GiB',
},
},
{
'name': 'ior',
'needed_key': 'RKBC',
'plotly_settings': {
'title': 'read io {host}',
'xtitle': 'Date',
'ytitle': 'I/O MiB/s',
'ytype': 'log',
},
},
{
'name': 'iow',
'needed_key': 'WKBC',
'plotly_settings': {
'title': 'write io {host}',
'xtitle': 'Date',
'ytitle': 'I/O MiB/s',
'ytype': 'log',
},
},
{
'name': 'ios',
'needed_key': 'syscalls',
'plotly_settings': {
'title': 'I/O syscalls {host}',
'xtitle': 'Date',
'ytitle': 'I/O syscalls/s',
'ytype': 'log',
},
},
]
### Name parse configs
# dict of all commands where the name will be filtered out
# key : command
# value: list of parameter that take a value what is not the name
NAME_SPEZIAL_PARAMETER_CONFIG = {
'java': ['-cp', '-classpath'],
'bash': [],
'sh': [],
'perl': [],
'python': ['-m', '-W', '-X', '--check-hash-based-pycs', '-c'],
}
### Filter Configs
# registered filter functions
......@@ -122,19 +16,4 @@ DEFAULT_FILTER = 'average'
### other Configs
# list of commands that are ignored
COMAND_BLACKLIST_REGEX = [
'collectl',
]
# list of colors of the plot entrys
PLOTLY_COLORS=['rgb(31, 119, 180)', 'rgb(255, 127, 14)',
'rgb(44, 160, 44)', 'rgb(214, 39, 40)',
'rgb(148, 103, 189)', 'rgb(140, 86, 75)',
'rgb(227, 119, 194)', 'rgb(127, 127, 127)',
'rgb(188, 189, 34)', 'rgb(23, 190, 207)']
### Merge configs
# hint: merger function must be in value_merger.py
# Defaults merge infos
# default_value is the startvalue for merging if no is given in NEEDED_VALUES
# default_merger is the merger function that is called if no is given in NEEDED_VALUES
NEEDED_VALUES_DEFAULTS = {
'default_value': 0,
'default_merger': 'addition_int',
}
# Information which values needed for plots
# key : name of value needed for plotconfig
# value: is dict with following configuration options
# 'keys' : must be a list with the table heads from collectl that is to be merge
# 'merger' : function that merge the values from the given keys to the new for the plots
# 'default': startvalue for merging
NEEDED_VALUES = {
'PCT': {'keys':['PCT']},
'VmRSS': {'keys':['VmRSS'], 'merger': 'x2oneaddition_float_sizedown2'},
'RKBC': {'keys':['RKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'WKBC': {'keys':['WKBC'], 'merger': 'x2oneaddition_float_sizedown1'},
'syscalls': {'keys':['RSYS', 'WSYS'], 'merger': 'x2oneaddtion_int'},
}
### Plots Config
# List of diagrams to be created
# config of Plot
# 'name': Is the name of the Plot will displayed on the website
# 'needed_key': is a key defined in NEEDED_VALUES for the values that are to be used in der Plot as y
# 'plotly_settings': are plotly configs to desing plots see doku for detail
# hint {host} will be replaced with real hostname in title and in xaxis, yaxis title
barchart_buttons = {
'annotations': [
{'text': 'time interval in Barchart:',
'x': 0.2, 'y': 1.13, 'yref':'paper', 'xref':'paper', 'showarrow':False}
],
'updatemenus':[
{
'type': 'buttons',
'x': 0.1,
'y': 1.15,
'direction': 'left',
'buttons': [
{'args':[{'type': 'scattergl'}, {}],
'label':'dots',
'method':'update'},
{'args':[{'type': 'histogram', 'histfunc': 'sum'}, {'barmode': 'stack'}],
'label':'barchart',
'method':'update'},
],
'showactive':True,
},
{
'active': 1,
'x': 0.4,
'y': 1.15,
'showactive': True,
'buttons' : [
{
'args': ['xbins.size', '1000'],
'label': '1 sec',
'method': 'restyle',
},
{
'args': ['xbins.size', '5000'],
'label': '5 sec',
'method': 'restyle',
},
{
'args': ['xbins.size', '30000'],
'label': '30 sec',
'method': 'restyle',
},
{
'args': ['xbins.size', '60000'],
'label': '1 min',
'method': 'restyle',
},
{
'args': ['xbins.size', '300000'],
'label': '5 min',
'method': 'restyle',
},
{
'args': ['xbins.size', '1800000'],
'label': 'half Hour',
'method': 'restyle',
},
{
'args': ['xbins.size', '3600000'],
'label': 'Hour',
'method': 'restyle',
},
{
'args': ['xbins.size', '43200000'],
'label': 'half Day',
'method': 'restyle',
},
]
}
],
}
PLOT_CONFIG = [{
'name': 'cpu',
'needed_key': 'PCT',
'plotly_settings': {
'data': {
'type': 'scattergl',
'mode': 'markers',
},
'layout': {
'height': 500,
'title': 'CPU load {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'CPU load'},
**barchart_buttons
},
},
},
{
'name': 'ram',
'needed_key': 'VmRSS',
'plotly_settings': {
'data': {
'type': 'scattergl',
'mode': 'markers',
},
'layout': {
'height': 500,
'title': 'Memory Usage {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'RAM usage GiB'},
**barchart_buttons
},
},
},
{
'name': 'ior',
'needed_key': 'RKBC',
'plotly_settings': {
'data': {
'type': 'scattergl',
'mode': 'markers',
},
'layout': {
'height': 500,
'title': 'read io {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'I/O MiB/s', 'type': 'log'},
**barchart_buttons
},
},
},
{
'name': 'iow',
'needed_key': 'WKBC',
'plotly_settings': {
'data': {
'type': 'scattergl',
'mode': 'markers',
},
'layout': {
'height': 500,
'title': 'write io {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'I/O MiB/s', 'type': 'log'},
**barchart_buttons
},
},
},
{
'name': 'ios',
'needed_key': 'syscalls',
'plotly_settings': {
'data': {
'type': 'scattergl',
'mode': 'markers',
},
'layout': {
'height': 500,
'title': 'I/O syscalls {host}',
'xaxis': {'title': 'Date'},
'yaxis': {'title': 'I/O syscalls/s', 'type': 'log'},
**barchart_buttons
},
},
},
]
### Name parse configs
# dict of all commands where the name will be filtered out
# key : command
# value: list of parameter that take a value what is not the name
NAME_SPEZIAL_PARAMETER_CONFIG = {
'java': ['-cp', '-classpath'],
'bash': [],
'sh': [],
'perl': [],
'python': ['-m', '-W', '-X', '--check-hash-based-pycs', '-c'],
}
### other Configs
# list of commands that are ignored
COMAND_BLACKLIST = [
'collectl',
]
# list of colors of the plot entrys
PLOTLY_COLORS=['rgb(31, 119, 180)', 'rgb(255, 127, 14)',
'rgb(44, 160, 44)', 'rgb(214, 39, 40)',
'rgb(148, 103, 189)', 'rgb(140, 86, 75)',
'rgb(227, 119, 194)', 'rgb(127, 127, 127)',
'rgb(188, 189, 34)', 'rgb(23, 190, 207)']
......@@ -24,7 +24,7 @@ yattag=<1.13.2
| -c, --collectl | how to call collectl | is the collectl that will be called to get the data from the sources | collectl without a path |
| -p, --plotlypath | a path to a plotly javascript libery | is needed for the plot in the html files | ./plotly-latest.min.js |
| -d, --destination | a path to a directory | is where a directory with the html files will be created | current directory |
| --shorten /<br> --notshorten | - | enable or disable shorten of commands with parameter/options only to file/command names. <br> examples:<br> python ~/scripts/script.py 1 --> script.py <br>ls -lisa --> ls | enabled |
| --shorten /<br> --notshorten | - | enable or disable shorten of commands with parameters/options only to file/command names. <br> examples:<br> python ~/scripts/script.py 1 --> script.py <br>ls -lisa --> ls | enabled |
| --coarsest /<br> --notcoarsest | - | enable or disable shorten of commands only to command names.<br> If enabled --shorten is ignored!<br> examples: <br> python ~/scripts/script.py 1 --> python <br> ls -lisa --> ls | disabled |
| --filtercmds / --notfiltercmds | - | enable or disable filtering | enabled |
| --filtertype | filtertype <br> see --help for detail | to select which filter to be used | see --help |
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment