Commit 20dad27f authored by Julius Metz's avatar Julius Metz

make python package, add relative axis support

parent 8125f0cd
......@@ -15,9 +15,9 @@ import click
from yattag import Doc
import filter_func
import value_merger
import Collectl2plotly_Plots_Config as default_plot_conf
import Collectl2plotly.filter_func as filter_func
import Collectl2plotly.value_merger as value_merger
import Collectl2plotly.Collectl2plotly_Plots_Config as default_plot_conf
### Filter Configs
......@@ -38,6 +38,30 @@ DEFAULT_FILTER = 'average'
CONFIG_VARIABLES = ['NEEDED_VALUES_DEFAULTS', 'NEEDED_VALUES', 'PLOTS_CONFIG',
'NAME_SPEZIAL_PARAMETER', 'COMAND_BLACKLIST', 'PLOTLY_COLORS', 'PLOTLY_STATIC_COLORS']
### Other
# config for find best axis interval and unit
# from smallest unit to greatest
# intervals = possible intervals of the axis(needed)
# max = conversion value if not given greatest supported unit
# unit = str with unit name
AXIS_INTERVAL_REL = [
{
'max': 60,
'intervals': [1, 5, 10],
'unit': 'sec',
},
{
'max': 60,
'intervals': [0.5, 1, 5, 10],
'unit': 'min',
},
{
'intervals': [0.25 ,0.5, 1, 5, 10],
'unit': 'h',
}
]
#pattern for split of commands (with \" and \ )
FIELDS_PATTERN = re.compile(r'(?:(?:\s*[^\\]|\A)\"(.*?[^\\]|)\"|(?:\s+|\A)(?=[^\s])(.*?[^\\])(?= |\Z))')
#(?:\"(.*?)\"|(\S+))
......@@ -220,9 +244,43 @@ def parse_file(path, collectl, shorten_cmds, coarsest, config):
return entry_data_plotfriendly, plot_filter_data
def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key):
def make_relative_xaxi(all_values):
"""build lists with relative values for xaxis
Arguments:
all_values {[datetime, ..]} -- list of all datetime objects for plot
Returns:
(list, list) -- 1.: list with the relative values that are displayed on the xaxis
2.: list with the datetime as str of the absolute values of xaxis
"""
min_value = min(all_values)
max_value = max(all_values)
guideline_tickcounts = 10
end_value = (max_value - min_value).total_seconds()
conversion_factor = 1
for current_unit in AXIS_INTERVAL_REL:
if 'max' in current_unit and end_value > current_unit['max']:
end_value /= current_unit['max']
conversion_factor *= current_unit['max']
continue
distances = [
max(end_value // interval - guideline_tickcounts, guideline_tickcounts - end_value // interval) \
for interval in current_unit['intervals']
]
interval = current_unit['intervals'][distances.index(min(distances))]
xaxis_ticks = [interval * i for i in range(int(end_value/interval)+1)]
if len(xaxis_ticks) == 1:
xaxis_ticks.append(1)
xaxis_ticks_values = [str(min_value + datetime.timedelta(seconds=tick * conversion_factor)) for tick in xaxis_ticks]
return xaxis_ticks, xaxis_ticks_values, current_unit['unit']
def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key, relative_xaxis):
"""Build a list of dicts in Plotlyconf style for the diffrent traces with the data from cmds_data of needed_key
than make plot dict in Plotly style and change '{host}' in titels. After that build plot as html div via Plotly.
than make plot dict in Plotly style and change '{host}' in titels.
Arguments:
cmds_data {dict} -- is the data for the plot example: {'Command': {'datetime':[...], 'cpu': [...]}}
......@@ -231,12 +289,12 @@ def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_va
plot_settings {dict} -- settings for Plotly
plotly_format_vars {dict} -- values for Plotly settings
needed_key {str} -- key of cmds_data for the values to be use
relative_xaxis {bool} -- if true add buttons to change xaxis to relative
Returns:
str -- plotly html div
dict -- with plotly jsons
"""
plot_data = []
#startdate = min([date for cmd in filter_info[needed_key] for date in cmds_data[cmd]['datetime']])
for cmd in filter_info[needed_key]:
plot_data.append({
'x': [str(date) for date in cmds_data[cmd]['datetime']],
......@@ -250,7 +308,9 @@ def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_va
)
layout = pickle.loads(pickle.dumps(plot_settings['layout'], -1))
layout = pickle.loads(pickle.dumps(plot_settings.get('layout', {}), -1))
# replace {host} in titles
if 'title' in layout:
if type(layout['title']) == str:
layout['title'] = layout['title'].format(
......@@ -270,18 +330,45 @@ def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_va
layout['yaxis']['title']['text'] = layout['yaxis']['title']['text'].format(
**plotly_format_vars,
)
xtitle = ''
if 'xaxis' in layout and 'title' in layout['xaxis']:
if type(layout['xaxis']['title']) == str:
layout['xaxis']['title'] = layout['xaxis']['title'].format(
**plotly_format_vars,
)
xtitle = layout['xaxis']['title']
else:
layout['xaxis']['title']['text'] = layout['xaxis']['title']['text'].format(
**plotly_format_vars,
)
xtitle = layout['xaxis']['title']['text']
return {'data': json.dumps(plot_data), 'layout': json.dumps(layout)}
# add buttons for switching xaxis if 'relative-absolute_xaxis' in PLOT_CONF is set.
if relative_xaxis:
ticks_name, ticks_value, unit = make_relative_xaxi(
[date for cmd in filter_info[needed_key] for date in cmds_data[cmd]['datetime']],
)
if not 'xaxis' in layout:
layout['xaxis'] = {}
if not 'updatemenus' in layout:
layout['updatemenus'] = []
layout['updatemenus'].append({
'type': 'buttons',
'x': 0.6,
'y': 1.15,
'direction': 'left',
'buttons': [
{'args':[{'xaxis':{'title': xtitle, 'tickvals': None}}],
'label':'absolute xaxis',
'method':'relayout'},
{'args':[{'xaxis': {'title': 'runtime in {}'.format(unit), 'tickvals': ticks_value, 'ticktext': ticks_name}}],
'label':'relative xaxis',
'method':'relayout'},
],
'showactive':True,
})
return {'data': json.dumps(plot_data), 'layout': json.dumps(layout), 'config': json.dumps(plot_settings.get('config', {}))}
def build_html(plots_dict):
......@@ -299,22 +386,12 @@ def build_html(plots_dict):
with tag('head'):
with tag('script', src='plotly.js'):
pass
with tag('script'):
doc.asis('''toggleplot = (name, index) => {
const elems = document.getElementsByClassName(name);
if (window.getComputedStyle(elems[index]).display === "none") {
elems[index].style.display = 'contents';
} else {
elems[index].style.display = 'none';
}
}''')
with tag('script'):
doc.asis("document.onreadystatechange = () => {if (document.readyState === 'complete') {")
doc.asis('console.log("test");')
for name, plots in plots_dict.items():
for i, plot in enumerate(plots):
doc.asis("Plotly.newPlot(document.getElementById('{name}-plot-{number}'), JSON.parse('{data}'), JSON.parse('{layout}'));"\
.format(name=name, number=i, data=plot['data'], layout=plot['layout'])
doc.asis("Plotly.newPlot(document.getElementById('{name}-plot-{number}'), JSON.parse('{data}'), JSON.parse('{layout}'), JSON.parse('{config}'));"\
.format(name=name, number=i, data=plot['data'], layout=plot['layout'], config=plot['config'])
)
doc.asis('}}')
with tag('body'):
......@@ -328,12 +405,7 @@ def build_html(plots_dict):
text(name)
for name, plots in plots_dict.items():
with tag('div', id=name):
#with tag('h2'):
# text(name)
for i in range(len(plots)):
#with tag('p'):
# with tag('button', onclick='toggleplot("{}", {})'.format(name, i)):
# text('toggle plot')
with tag('div', id='{}-plot-{}'.format(name, i)):
pass
......@@ -398,7 +470,7 @@ def data_from_file(arguments):
@click.option('--destination', '-d', default='.', type=click.Path(exists=True), show_default=True, help='path to directory where directory with plots will be created')
@click.option('--configpath', default=None, type=click.Path(exists=True), help='python file with plot and merge infos see doku for detail')
@click.option('--shorten/--notshorten', default=True, help='commands will be shorted only to name')
@click.option('--coarsest/--notcoarsest', default=False, help='commands will be shorted only to type (bash, perl, ...)')
@click.option('--coarsest', is_flag=True, help='commands will be shorted only to type (bash, perl, ...)')
@click.option('--filtercmds/--notfiltercmds', default=True, help='filtering or not')
@click.option('--filtervalue', help='Parameter which is given to the filter.')
@click.option('--filtertype',
......@@ -430,7 +502,7 @@ def main(source, collectl, plotlypath, destination, configpath,
config_module = default_plot_conf
if configpath:
if configpath.endswith('.py'):
config_module = importlib.import_module(configpath[:-3])
config_module = importlib.machinery.SourceFileLoader('config', configpath).load_module()
else:
print('given config isn`t a ".py" Python file')
exit(1)
......@@ -484,6 +556,7 @@ def main(source, collectl, plotlypath, destination, configpath,
plot_config['plotly_settings'],
{'host': host},
plot_config['needed_key'],
plot_config.get('relative-absolute_xaxis', False),
))
print("plots build in {:.1f}s".format(time.time() - start_plots_build))
......
......@@ -108,6 +108,7 @@ barchart_buttons = {
PLOTS_CONFIG = [{
'name': 'CPU load',
'needed_key': 'PCT',
'relative-absolute_xaxis': True,
'plotly_settings': {
'data': {
'type': 'scattergl',
......@@ -121,11 +122,21 @@ PLOTS_CONFIG = [{
'showlegend': True,
**barchart_buttons
},
'config': {
'toImageButtonOptions': {
'format': 'svg',
'filename': 'cpu_load_plot',
'height': 900,
'width': 1600,
'scale': 1,
},
},
},
},
{
'name': 'RAM usage',
'needed_key': 'VmRSS',
'relative-absolute_xaxis': True,
'plotly_settings': {
'data': {
'type': 'scattergl',
......@@ -139,11 +150,21 @@ PLOTS_CONFIG = [{
'showlegend': True,
**barchart_buttons
},
'config': {
'toImageButtonOptions': {
'format': 'svg',
'filename': 'ram_usage_plot',
'height': 900,
'width': 1600,
'scale': 1,
},
},
},
},
{
'name': 'I/O read',
'needed_key': 'RKBC',
'relative-absolute_xaxis': True,
'plotly_settings': {
'data': {
'type': 'scattergl',
......@@ -157,11 +178,21 @@ PLOTS_CONFIG = [{
'showlegend': True,
**barchart_buttons
},
'config': {
'toImageButtonOptions': {
'format': 'svg',
'filename': 'io_read_plot',
'height': 900,
'width': 1600,
'scale': 1,
},
},
},
},
{
'name': 'I/O write',
'needed_key': 'WKBC',
'relative-absolute_xaxis': True,
'plotly_settings': {
'data': {
'type': 'scattergl',
......@@ -175,11 +206,21 @@ PLOTS_CONFIG = [{
'showlegend': True,
**barchart_buttons
},
'config': {
'toImageButtonOptions': {
'format': 'svg',
'filename': 'io_write_plot',
'height': 900,
'width': 1600,
'scale': 1,
},
},
},
},
{
'name': 'I/O syscalls',
'needed_key': 'syscalls',
'relative-absolute_xaxis': True,
'plotly_settings': {
'data': {
'type': 'scattergl',
......@@ -193,6 +234,15 @@ PLOTS_CONFIG = [{
'showlegend': True,
**barchart_buttons
},
'config': {
'toImageButtonOptions': {
'format': 'svg',
'filename': 'io_syscalls_plot',
'height': 900,
'width': 1600,
'scale': 1,
},
},
},
},
]
......
from .Collectl2plotly import main
\ No newline at end of file
from setuptools import setup, find_packages
setup(name='Collectl2plotly',
version='1',
author='TU Dresden',
python_requires=">=3.6",
packages=find_packages(),
package_data={'': ['plotly-latest.min.js']},
scripts=[],
entry_points='''
[console_scripts]
Collectl2plotly=Collectl2plotly:main
''',
install_requires=[
'yattag',
'click',
],
)
......@@ -2,18 +2,17 @@
# Collectl2plotly
## How to
For use you need to install the requirements via pip.
After that you can use it like that:
For use you need to install it via pip:
```
python Collectl2plotly <options>
pip install ./Collectl2plotly
```
For information of the option use "--help" or go to the option description.
### Requirements
After that you can use it like that:
```
Click=<7.0
yattag=<1.13.2
Collectl2plotly <options>
```
For informations of the options use "--help" or go to the [options](#options) description.
## Options
......@@ -25,7 +24,7 @@ yattag=<1.13.2
| -d, --destination | a path to a directory | is where a directory with the html files will be created | current directory |
| --configpath | a path to config file | is a python file with the plot and merge settings see [here](#config)| a default config |
| --shorten /<br> --notshorten | - | enable or disable shorten of commands with parameters/options only to file/command names. <br> examples:<br> python ~/scripts/script.py 1 --> script.py <br>ls -lisa --> ls | enabled |
| --coarsest /<br> --notcoarsest | - | enable or disable shorten of commands only to command names.<br> If enabled --shorten is ignored!<br> examples: <br> python ~/scripts/script.py 1 --> python <br> ls -lisa --> ls | disabled |
| --coarsest | - | enable shorten of commands only to command names.<br> If enabled --shorten is ignored!<br> examples: <br> python ~/scripts/script.py 1 --> python <br> ls -lisa --> ls | disabled |
| --filtercmds / --notfiltercmds | - | enable or disable filtering | enabled |
| --filtertype | filtertype <br> see --help for detail | to select which filter to be used | see --help |
| --filtervalue | any value (string, int, ...) | is passed to the filter function as string | - |
......@@ -125,6 +124,10 @@ Is the name of the Plot this will be displayed in the index of the websites.
**'needed_key'**:<br>
Is a name from NEEDED_VALUES this values are used as y axis data in the plot.
**'relative-absolute_xaxis'**:<br>
Is a bool if True add buttons to the plot that switch the xaxis from absolute Date to Runtime and back.
If not given it is False.
**'plotly_settings'**:<br>
Is a settings dict in Plotly style, here it is possible to set everything about the appearance of the plots.<br>
Hints for plotly_settings:
......@@ -138,6 +141,7 @@ PLOTS_CONFIG = [
{
'name': 'CPU load',
'needed_key': 'CPU',
'relative-absolute_xaxis': True,
'plotly_settings': {
'data': {
'type': 'scattergl',
......@@ -277,7 +281,7 @@ A merger is a function that merge a values from collectl to existing base value.
| possible argument | take | describtion | supported merger |
| ----------------- | ---- | ----------- | ---------------- |
| round | int | Rounding merged collectl values on given point (Python round function). | x2oneaddition_* |
| operator | basic python operater as str (/ , * , ...) | Specifies which operator should be used to calculate a value from parameter operator_value and the collectl values who already merged. This will be merged to the base value. | x2oneaddition_* |
| operator | basic python operater as str (/ , * , -, +, %) | Specifies which operator should be used to calculate a value from parameter operator_value and the collectl values who already merged. This will be merged to the base value. | x2oneaddition_* |
| operator_value | int/float | value that must be given to use operator for use see operator. | x2oneaddition_* |
### Add new Merger
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment