Collectl2plotly.py 31.9 KB
Newer Older
1
import re
Julius Metz's avatar
Julius Metz committed
2
import gzip
Julius Metz's avatar
Julius Metz committed
3 4
import subprocess
from pathlib import Path
5 6
import copy
import time
7
import datetime
Julius Metz's avatar
Julius Metz committed
8 9
import multiprocessing as mp
import shutil
Julius Metz's avatar
Julius Metz committed
10
import importlib
Julius Metz's avatar
Julius Metz committed
11 12
import pickle
import json
Julius Metz's avatar
Julius Metz committed
13 14

import click
15 16
from yattag import Doc

Julius Metz's avatar
Julius Metz committed
17

18 19 20
import Collectl2plotly.filter_func as filter_func
import Collectl2plotly.value_merger as value_merger
import Collectl2plotly.Collectl2plotly_Plots_Config as default_plot_conf
Julius Metz's avatar
Julius Metz committed
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36


### Filter Configs

# registered filter functions
# filter functions must be in filter_func.py
# key  : Display name
# value: function name
FILTER_FUNCTIONS = {'hardvalue': 'filter_hardvalue', 'average': 'filter_average'}

# Default filter function if no explicit is given
# must be a display name of a function
DEFAULT_FILTER = 'average'

### validation Config

Julius Metz's avatar
Julius Metz committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
# Merger that exist and posible kwargs/parameter
MERGER = {
    'x2oneaddition_int': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
    },
    'x2oneaddition_float': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
    }
}

# validation infos of config
# key = config variablename to validate
# value = validator dict or type/tuple of types
# hints!!!!
# '__merger__' can be used as type for the merger tuple.
# in fixed_keys and 'changing_keys' the 'validator' need not be given if sub-elements are not to be validated.
#
# validator dict has follow structur:
# 'type' : required, is the possible type/s of the obj that are valid
# 'emptyable': list/dict/tuple can be empty if emptyable is True. Default: False
# 'subvalue_validator': is a validatordict or type/tuple of types to check all elems are valid if type is list/tuple
# 'fixed_keys': only if type is dict. takes a dict:
#               key: a known key of the current dict that is checked
#               value: dict with following possible keys:
#                     'required': if False the key must not in the dict Default: True
#                     'validator':  is a validatordict or type/tuple of types to validate the value
# 'changing_keys': only if type is dict. takes a dict:
#               'key_type': type of the keys in the dict current dict
#               'validator':  is a validatordict or type/tuple of types to validate the values of the dict
#
VALIDATIONS_INFOS = {
    'NEEDED_VALUES_DEFAULTS': {
        'type': dict,
        'fixed_keys': {
            'default_base_value': {},
            'default_merger': {'validator': '__merger__'},
        },
    },
    'NEEDED_VALUES': {
        'type': dict,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': dict,
                'fixed_keys': {
                    'keys': {'validator': {'type': list, 'subvalue_validator': str}},
                    'merger': {'validator': '__merger__'},
                    'base_value': {'required': False}
                },
            },
        },
    },
    'PLOTS_CONFIG': {
        'type': list,
        'subvalue_validator': {
            'type': dict,
            'fixed_keys': {
                'name': {'validator': str},
                'needed_key': {'validator': str},
                'relative-absolute_xaxis': {'required': False, 'validator': bool},
                'plotly_settings': {'validator': dict},
            },
        },
    },
    'NAME_SPEZIAL_PARAMETER': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': list,
                'emptyable':True,
                'subvalue_validator': str,
            },
        },
    },
    'COMAND_BLACKLIST': {
        'type': list,
        'emptyable': True,
        'subvalue_validator': str,
    },
    'PLOTLY_COLORS': {
        'type': list,
        'subvalue_validator': str,
    },
    'PLOTLY_STATIC_COLORS': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': str,
        },
    },
}

Julius Metz's avatar
WC  
Julius Metz committed
136

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
### Other

# config for find best axis interval and unit
# from smallest unit to greatest
# intervals = possible intervals of the axis(needed)
# max = conversion value if not given greatest supported unit
# unit = str with unit name
AXIS_INTERVAL_REL = [
    {
        'max': 60,
        'intervals': [1, 5, 10],
        'unit': 'sec',
    },
    {
        'max': 60,
        'intervals': [0.5, 1, 5, 10],
        'unit': 'min',
    },
    {
        'intervals': [0.25 ,0.5, 1, 5, 10],
        'unit': 'h',
    }
]

Julius Metz's avatar
Julius Metz committed
161 162 163
#pattern for split of commands (with \" and \ )
FIELDS_PATTERN = re.compile(r'(?:(?:\s*[^\\]|\A)\"(.*?[^\\]|)\"|(?:\s+|\A)(?=[^\s])(.*?[^\\])(?= |\Z))')
#(?:\"(.*?)\"|(\S+))
164 165


Julius Metz's avatar
Julius Metz committed
166 167 168 169 170 171 172 173 174 175

def is_merger(to_check):
    """Check is given obj a valid merger. Possible merger are given in MERGER.

    Arguments:
        to_check {unkown} -- obj to test

    Returns:
        (bool, str) -- return true if it is a merger false and error if not
    """
176 177

    # check is tuple with 2 elems
Julius Metz's avatar
Julius Metz committed
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
    try:
        name, mergerkwargs = to_check
    except ValueError:
        return False, "'{0}' has no right merger structur expected: (name[str], kwargs[dict])".format(to_check)

    if isinstance(name, str) and isinstance(mergerkwargs, dict):
        if name in MERGER:
            for kwarg_name, value in mergerkwargs.items():
                if kwarg_name in MERGER[name]:
                    if 'choices' in MERGER[name][kwarg_name]:
                        if not value in MERGER[name][kwarg_name]['choices']:
                            return False, '{0} {1} expect one of {2} got: {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['choices'], value
                            )
                    else:
                        if not isinstance(value, MERGER[name][kwarg_name]['type']):
                            return False, '{1} of {0} expect {2} not {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['type'], type(value)
                            )
                else:
                    return False, '{1} is unknown parameter of {0}'.format(name, kwarg_name)
        else:
            return False, '"{0}" is unknown merger'.format(name)
    else:
        return False, '"{0}" has no right merger structur expected: (name[str], kwargs[dict])'.format(to_check)

    return True, None


def check_valid(to_check, validationinfos):
    """check object is valid from the perspective of validationinfos

    Arguments:
        to_check {unkown} -- object to check is valid
        validationinfos {dict or type or tuple with types} -- infos what is  needed that to_check is valid

    Returns:
        (bool, str) -- return true if it is valid, false and error if not
    """
    if isinstance(validationinfos, dict):
        if validationinfos['type'] == '__merger__':
            return is_merger(to_check)
        else:
            if isinstance(to_check, validationinfos['type']):
222
                if type(to_check) == dict:
Julius Metz's avatar
Julius Metz committed
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

                    for fixed_name, infos in validationinfos.get('fixed_keys', {}).items():
                        if fixed_name in to_check:
                            if 'validator' in infos:
                                valid, error_msg = check_valid(to_check[fixed_name], infos['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(fixed_name, error_msg)

                        elif infos.get('required', True):
                            return False, '{0} missing key: "{1}"'.format(to_check, fixed_name)

                    if 'changing_keys' in validationinfos:
                        for key, subvalue in to_check.items():
                            if not isinstance(key, validationinfos['changing_keys']['key_type']):
                                return False, '{0} key has wrong type expected {1} got {2}'.format(
                                    key,  validationinfos['changing_keys']['key_type'], type(key),
                                )
                            if 'validator' in validationinfos['changing_keys']:
                                valid, error_msg =  check_valid(subvalue, validationinfos['changing_keys']['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(key, error_msg)

247
                if isinstance(to_check, (list, tuple)):
Julius Metz's avatar
Julius Metz committed
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

                    if 'subvalue_validator' in validationinfos:
                        for i, subvalue in enumerate(to_check):
                            valid, error_msg = check_valid(subvalue, validationinfos['subvalue_validator'])
                            if not valid:
                                return valid, '[{0}] => {1}'.format(i, error_msg)

            else:
                return False, '{0} is wrong type expected {1} got {2}'.format(
                        to_check, validationinfos['type'], type(to_check)
                    )

    else:
        if validationinfos == '__merger__':
            return is_merger(to_check)
        else:
            if not isinstance(to_check, validationinfos):
                return False, '{0} is {1} expected: {2}'.format(
                    to_check, type(to_check), validationinfos,
                )

    return True, None


274
def datestr2date(datestr):
Julius Metz's avatar
Julius Metz committed
275 276 277 278 279 280 281 282
    """Converts a "datestring" to a date Object.

    Arguments:
        datestr {str} -- string of a Date example: 20191224

    Returns:
        datetime.date -- date of the string
    """
283
    return datetime.date(
Julius Metz's avatar
Julius Metz committed
284 285 286 287
        int(datestr[:4]),
        int(datestr[4:6]),
        int(datestr[6:8]),
    )
288 289


Julius Metz's avatar
Julius Metz committed
290
def get_cmdname(cmd, spezial_parameters_of_all, coarsest=False):
291 292 293 294 295 296 297 298 299 300 301
    """search in complete commandstring the name of the skript or the command that is used

    Arguments:
        cmd {str} -- complete commandstring

    Keyword Arguments:
        coarsest {bool} -- return only the call function(example: bash, python) if True (default: {False})

    Returns:
        str -- new cmd name
    """
Matthias Lieber's avatar
Matthias Lieber committed
302
    # search for (for example) bash_function="python" in cmd="/usr/bin/python3 my_script.py"
Julius Metz's avatar
Julius Metz committed
303 304 305
    # split command with regex
    cmd_splited = [x[0] or x[1] for x in FIELDS_PATTERN.findall(cmd)]
    #shlex.split(cmd)
306 307
    bash_function = cmd_splited[0].split('/')[-1]
    bash_function = re.search(r'[^\W\n]+', bash_function).group(0)
Matthias Lieber's avatar
Matthias Lieber committed
308
    # check if bash_function is known, if not, return bash_function
Julius Metz's avatar
Julius Metz committed
309 310
    spezial_parameters = spezial_parameters_of_all.get(bash_function, None)
    if coarsest or spezial_parameters is None:
311 312
        return bash_function
    skip = False
Matthias Lieber's avatar
Matthias Lieber committed
313
    # search script/program name within the parameters and only return this without path
314 315 316 317
    for position, parameter in enumerate(cmd_splited[1:]):
        if skip:
            skip = False
            continue
Julius Metz's avatar
Julius Metz committed
318
        if parameter in spezial_parameters:
319 320 321 322
            skip = True
            continue
        if bash_function == 'bash' or bash_function == 'sh' and parameter == '-c':
            return bash_function + ' -c'
Julius Metz's avatar
Julius Metz committed
323
            # return cmd_splited[position+1]
324 325 326 327 328 329
        if parameter.startswith('-'):
            continue
        return parameter.split('/')[-1]
    return cmd


Julius Metz's avatar
Julius Metz committed
330
def parse_file(path, collectl, shorten_cmds, coarsest, config):
Julius Metz's avatar
Julius Metz committed
331 332 333 334 335 336 337 338
    """start subproccess collectl than parse the output and merge.
       After that build usefull dict from parsed data

    Arguments:
        path {Path} -- path to collectl file to parse
        collectl {str} -- collectl command
        shorten_cmds {bool} -- if True cmd will be shorted by get_cmdname
        coarsest {bool} -- parameter of get_cmdname
Julius Metz's avatar
Julius Metz committed
339 340
        config {dict} -- plots and merge Config

Julius Metz's avatar
Julius Metz committed
341 342

    Returns:
Julius Metz's avatar
Julius Metz committed
343 344 345
        (dict, dict) --  1. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                         2. data for filter function :
                         {'number_of_values': X, 'commands':{ cmd:{'number_of_values': X, metrics:SUM, ...}}, metrics:SUM, ...}
Julius Metz's avatar
Julius Metz committed
346
    """
347
    collectl_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
348
    # run collectl in playback mode and read output into list
349
    process = subprocess.run(
Matthias Lieber's avatar
Matthias Lieber committed
350
        [collectl, '-P', '-p', path, '-sZ'], stdout=subprocess.PIPE, check=True,
351
    )
Julius Metz's avatar
Julius Metz committed
352 353
    print('collectl make table took {:.1f}s'.format(
        time.time() - collectl_starttime))
354
    parsing_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
355
    # output contains all data!
Julius Metz's avatar
Julius Metz committed
356
    output = process.stdout.decode().splitlines()
Matthias Lieber's avatar
Matthias Lieber committed
357
    # get table head
Julius Metz's avatar
Julius Metz committed
358
    head = output.pop(0).split(' ')
359 360 361 362 363 364 365
    for possible_head in output[:]:
        if possible_head.startswith('#'):
            head = possible_head.split(' ')
            output.remove(possible_head)
        else:
            break

Matthias Lieber's avatar
Matthias Lieber committed
366
    # get template of an entry from the head
Julius Metz's avatar
Julius Metz committed
367
    head[0] = head[0][1:]
Julius Metz's avatar
Julius Metz committed
368 369
    head_indexes_dict = {
        head_title: index for index, head_title in enumerate(head)}
370 371
    empty_dict_with_value_titles = {
        value_title: copy.deepcopy(
Julius Metz's avatar
Julius Metz committed
372
            value_title_settings.get(
Julius Metz's avatar
Julius Metz committed
373
                'base_value', config['NEEDED_VALUES_DEFAULTS']['default_base_value']
Julius Metz's avatar
Julius Metz committed
374
            )
Julius Metz's avatar
Julius Metz committed
375
        ) for value_title, value_title_settings in config['NEEDED_VALUES'].items()
376
    }
Julius Metz's avatar
Julius Metz committed
377

Matthias Lieber's avatar
Matthias Lieber committed
378 379
    # parse all output lines
    entrys_data = {}
Julius Metz's avatar
WC  
Julius Metz committed
380
    cmd_cmdshort_dict = {}
Julius Metz's avatar
Julius Metz committed
381 382
    merger_lookup_dict = {}

Julius Metz's avatar
Julius Metz committed
383
    for entry in output:
Matthias Lieber's avatar
Matthias Lieber committed
384
        # split by ' ' (exclude command from splitting)
385
        splited_entry = entry.split(' ', len(head_indexes_dict)-1)
Matthias Lieber's avatar
Matthias Lieber committed
386
        # get command string and shorten
Julius Metz's avatar
Julius Metz committed
387
        cmd = splited_entry[-1]
Julius Metz's avatar
Julius Metz committed
388
        if shorten_cmds or coarsest:
Julius Metz's avatar
WC  
Julius Metz committed
389 390 391
            if cmd in cmd_cmdshort_dict:
                cmd = cmd_cmdshort_dict[cmd]
            else:
Julius Metz's avatar
Julius Metz committed
392
                short_cmd = get_cmdname(cmd, config['NAME_SPEZIAL_PARAMETER'], coarsest=coarsest)
Julius Metz's avatar
WC  
Julius Metz committed
393 394
                cmd_cmdshort_dict[cmd]= short_cmd
                cmd = short_cmd
Julius Metz's avatar
Julius Metz committed
395
        if cmd in config['COMAND_BLACKLIST']:
Julius Metz's avatar
Julius Metz committed
396
            continue
Matthias Lieber's avatar
Matthias Lieber committed
397
        # create dict for each command
Julius Metz's avatar
Julius Metz committed
398 399
        if not cmd in entrys_data:
            entrys_data[cmd] = {}
Matthias Lieber's avatar
Matthias Lieber committed
400
        # get datetime obj for current entry
Julius Metz's avatar
Julius Metz committed
401 402
        tmp_datetime = datetime.datetime.combine(
            datestr2date(splited_entry[head_indexes_dict['Date']]),
Matthias Lieber's avatar
Matthias Lieber committed
403
            datetime.time(*[int(n) for n in splited_entry[head_indexes_dict['Time']].split(':')]),
Julius Metz's avatar
Julius Metz committed
404
        )
Matthias Lieber's avatar
Matthias Lieber committed
405
        # if datetime not yet existing, add new entry from template
Julius Metz's avatar
Julius Metz committed
406
        if not tmp_datetime in entrys_data[cmd]:
Julius Metz's avatar
Julius Metz committed
407 408
            entrys_data[cmd][tmp_datetime] = pickle.loads(pickle.dumps(
                empty_dict_with_value_titles, -1))
Matthias Lieber's avatar
Matthias Lieber committed
409 410
        # get values from data as given in NEEDED_VALUES and run specified merger function
        # to merge multiple values with same timestamp or rescale (e.g. to GB)
Julius Metz's avatar
Julius Metz committed
411 412 413 414 415
        for value_title, value_title_settings in config['NEEDED_VALUES'].items():
            merger, merger_kwargs = value_title_settings.get('merger', config['NEEDED_VALUES_DEFAULTS']['default_merger'])
            if not merger in merger_lookup_dict:
                merger_lookup_dict[merger] = getattr(value_merger, merger)
            entrys_data[cmd][tmp_datetime][value_title] = merger_lookup_dict[merger](
Julius Metz's avatar
Julius Metz committed
416 417
                entrys_data[cmd][tmp_datetime][value_title],
                *[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
Julius Metz's avatar
Julius Metz committed
418
                **merger_kwargs
419
            )
Julius Metz's avatar
Julius Metz committed
420
    print('parsing/merge took {:.1f}s'.format(time.time() - parsing_starttime))
421
    dictbuild_starttime = time.time()
422

Matthias Lieber's avatar
Matthias Lieber committed
423 424
    # create lists entry_data_plotfriendly[cmd][metric] = [ values, ... ] with the actual data to plot
    # and sum up all metrics for each command to enable filtering of non-interesting commands later on
425
    entry_data_plotfriendly = {}
Julius Metz's avatar
Julius Metz committed
426
    plot_filter_data = pickle.loads(pickle.dumps(empty_dict_with_value_titles, -1))
427 428 429
    plot_filter_data['number_of_values'] = 0
    plot_filter_data['commands'] = {}
    for cmd, cmd_data in entrys_data.items():
Julius Metz's avatar
Julius Metz committed
430 431
        plot_filter_data['commands'][cmd] = pickle.loads(pickle.dumps(
            empty_dict_with_value_titles, -1))
432
        plot_filter_data['commands'][cmd]['number_of_values'] = 0
Julius Metz's avatar
Julius Metz committed
433
        entry_data_plotfriendly[cmd] = {key: [] for key in config['NEEDED_VALUES']}
434
        entry_data_plotfriendly[cmd]['datetime'] = []
Julius Metz's avatar
Julius Metz committed
435
        for cmd_data_time, cmd_data_values in cmd_data.items():
436 437 438 439 440 441 442 443
            entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
            for cmd_data_key, cmd_data_value in cmd_data_values.items():
                entry_data_plotfriendly[cmd][cmd_data_key].append(cmd_data_value)

                plot_filter_data['commands'][cmd][cmd_data_key] += cmd_data_value
                plot_filter_data['commands'][cmd]['number_of_values'] += 1
                plot_filter_data[cmd_data_key] += cmd_data_value
                plot_filter_data['number_of_values'] += 1
Julius Metz's avatar
Julius Metz committed
444 445 446 447
    print(
        'data dict/ filter_data_dict build took {:.1f}s'.format(
            time.time() - dictbuild_starttime),
    )
448
    return entry_data_plotfriendly, plot_filter_data
Julius Metz's avatar
Julius Metz committed
449

Julius Metz's avatar
Julius Metz committed
450

451 452 453 454 455 456 457 458 459 460 461 462
def make_relative_xaxi(all_values):
    """build lists with relative values for xaxis

    Arguments:
        all_values {[datetime, ..]} -- list of all datetime objects for plot

    Returns:
        (list, list) -- 1.: list with the relative values that are displayed on the xaxis
                        2.: list with the datetime as str of the absolute values of xaxis
    """
    min_value = min(all_values)
    max_value = max(all_values)
463 464

    # Is the reference value where the next possible value is searched. For a good number of ticks on the axis.
465
    guideline_tickcounts = 10
466

467 468 469 470
    end_value = (max_value - min_value).total_seconds()
    conversion_factor = 1

    for current_unit in AXIS_INTERVAL_REL:
471
        #  Convert to higher units until the correct one is sufficient
472 473 474 475 476
        if 'max' in current_unit and end_value > current_unit['max']:
            end_value /= current_unit['max']
            conversion_factor *= current_unit['max']
            continue

477
        # find out the best possible interval
478 479 480 481 482
        distances = [
            max(end_value // interval - guideline_tickcounts, guideline_tickcounts - end_value // interval) \
                for interval in current_unit['intervals']
        ]
        interval = current_unit['intervals'][distances.index(min(distances))]
483 484

        # make array with plotly axis data
485 486 487 488 489 490 491 492
        xaxis_ticks = [interval * i for i in range(int(end_value/interval)+1)]
        if len(xaxis_ticks) == 1:
            xaxis_ticks.append(1)
        xaxis_ticks_values = [str(min_value + datetime.timedelta(seconds=tick * conversion_factor)) for tick in xaxis_ticks]
        return xaxis_ticks, xaxis_ticks_values, current_unit['unit']


def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key, relative_xaxis):
Julius Metz's avatar
WC  
Julius Metz committed
493
    """Build a list of dicts in Plotlyconf style for the diffrent traces with the data from cmds_data of needed_key
494
        than make plot dict in Plotly style and change '{host}' in titels.
Julius Metz's avatar
Julius Metz committed
495 496 497 498 499

    Arguments:
        cmds_data {dict} -- is the data for the plot example: {'Command': {'datetime':[...], 'cpu': [...]}}
        filter_info {dict} -- has the information which cmds will be shown example: {'cpu': [cmds]}
        cmd_color {dict} -- assign a fixed plot color to each cmd example:  {'Command': 'rgb(0, 0, 128)'}
Julius Metz's avatar
WC  
Julius Metz committed
500 501
        plot_settings {dict} -- settings for Plotly
        plotly_format_vars {dict} -- values for Plotly settings
Julius Metz's avatar
Julius Metz committed
502
        needed_key {str} -- key of cmds_data for the values to be use
503
        relative_xaxis {bool} -- if true add buttons to change xaxis to relative
Julius Metz's avatar
Julius Metz committed
504 505

    Returns:
506
        dict -- with plotly jsons
Julius Metz's avatar
Julius Metz committed
507 508
    """
    plot_data = []
Julius Metz's avatar
Julius Metz committed
509
    # make plotly plots config
Julius Metz's avatar
Julius Metz committed
510
    for cmd in filter_info[needed_key]:
Julius Metz's avatar
Julius Metz committed
511 512 513
        plot_data.append({
            'x': [str(date) for date in cmds_data[cmd]['datetime']],
            'y': cmds_data[cmd][needed_key],
Julius Metz's avatar
Julius Metz committed
514
            'name': cmd,
Julius Metz's avatar
Julius Metz committed
515
            'marker': {
Julius Metz's avatar
Julius Metz committed
516 517
                'color': cmd_color[cmd],
            },
Julius Metz's avatar
Julius Metz committed
518
            **plot_settings['data'],
Julius Metz's avatar
WC  
Julius Metz committed
519 520 521 522
        }
        )


523 524 525
    layout = pickle.loads(pickle.dumps(plot_settings.get('layout', {}), -1))

    # replace {host} in titles
Julius Metz's avatar
Julius Metz committed
526 527 528
    if 'title' in layout:
        if type(layout['title']) == str:
            layout['title'] = layout['title'].format(
Julius Metz's avatar
Julius Metz committed
529 530 531
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
532
            layout['title']['text'] = layout['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
533 534
                **plotly_format_vars,
            )
Julius Metz's avatar
WC  
Julius Metz committed
535

Julius Metz's avatar
Julius Metz committed
536 537 538
    if 'yaxis' in layout and 'title' in layout['yaxis']:
        if type(layout['yaxis']['title']) == str:
            layout['yaxis']['title'] = layout['yaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
539 540 541
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
542
            layout['yaxis']['title']['text'] = layout['yaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
543 544
                **plotly_format_vars,
            )
545
    xtitle = ''
Julius Metz's avatar
Julius Metz committed
546 547 548
    if 'xaxis' in layout and 'title' in layout['xaxis']:
        if type(layout['xaxis']['title']) == str:
            layout['xaxis']['title'] = layout['xaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
549 550
                **plotly_format_vars,
            )
551
            xtitle = layout['xaxis']['title']
Julius Metz's avatar
Julius Metz committed
552
        else:
Julius Metz's avatar
Julius Metz committed
553
            layout['xaxis']['title']['text'] = layout['xaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
554 555
                **plotly_format_vars,
            )
556
            xtitle = layout['xaxis']['title']['text']
Julius Metz's avatar
WC  
Julius Metz committed
557

558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583
    # add buttons for switching xaxis if 'relative-absolute_xaxis' in PLOT_CONF is set.
    if relative_xaxis:
            ticks_name, ticks_value, unit = make_relative_xaxi(
                [date for cmd in filter_info[needed_key] for date in cmds_data[cmd]['datetime']],
            )
            if not 'xaxis' in layout:
                layout['xaxis'] = {}
            if not 'updatemenus' in layout:
                layout['updatemenus'] = []
            layout['updatemenus'].append({
                'type': 'buttons',
                'x': 0.6,
                'y': 1.15,
                'direction': 'left',
                'buttons': [
                    {'args':[{'xaxis':{'title': xtitle, 'tickvals': None}}],
                    'label':'absolute xaxis',
                    'method':'relayout'},
                    {'args':[{'xaxis': {'title': 'runtime in {}'.format(unit), 'tickvals': ticks_value, 'ticktext': ticks_name}}],
                    'label':'relative xaxis',
                    'method':'relayout'},
                ],
                'showactive':True,
            })

    return {'data': json.dumps(plot_data), 'layout': json.dumps(layout), 'config': json.dumps(plot_settings.get('config', {}))}
Julius Metz's avatar
Julius Metz committed
584 585 586


def build_html(plots_dict):
Julius Metz's avatar
Julius Metz committed
587 588 589 590 591 592 593 594
    """build html site with the plots in plotsdict

    Arguments:
        plots_dict {dict} -- key: name of plot value: [plot, ...]

    Returns:
        str -- html website
    """
595 596 597 598
    doc, tag, text = Doc().tagtext()
    doc.asis('<!DOCTYPE html>')
    with tag('html'):
        with tag('head'):
Julius Metz's avatar
Julius Metz committed
599 600
            with tag('script', src='plotly.js'):
                pass
Julius Metz's avatar
Julius Metz committed
601 602 603 604
            with tag('script'):
                doc.asis("document.onreadystatechange = () => {if (document.readyState === 'complete') {")
                for name, plots in plots_dict.items():
                    for i, plot in enumerate(plots):
605 606
                        doc.asis("Plotly.newPlot(document.getElementById('{name}-plot-{number}'), JSON.parse('{data}'), JSON.parse('{layout}'), JSON.parse('{config}'));"\
                            .format(name=name, number=i, data=plot['data'], layout=plot['layout'], config=plot['config'])
Julius Metz's avatar
Julius Metz committed
607 608
                        )
                doc.asis('}}')
609 610 611 612 613
        with tag('body'):
            with tag('div', id='index'):
                with tag('h2'):
                    text('Index:')
                with tag('ul'):
Julius Metz's avatar
Julius Metz committed
614
                    for name in plots_dict:
615 616 617 618 619
                        with tag('li'):
                            with tag('a', href='#'+name):
                                text(name)
            for name, plots in plots_dict.items():
                with tag('div', id=name):
Julius Metz's avatar
Julius Metz committed
620 621 622
                    for i in range(len(plots)):
                        with tag('div', id='{}-plot-{}'.format(name, i)):
                            pass
623 624 625 626

    return doc.getvalue()


Julius Metz's avatar
Julius Metz committed
627 628 629 630 631 632 633 634 635 636 637 638 639
def get_sources(sources):
    """collect all sources (.raw.gz files)

    Arguments:
        sources {tuple} -- tuple of files/directorys

    Returns:
        list -- list of all .raw.gz files which was found
    """
    source_paths = []
    for source in sources:
        source_path =  Path(source)
        if source_path.is_dir():
Julius Metz's avatar
Julius Metz committed
640
            source_paths.extend(source_path.glob('*.raw.gz'))
Julius Metz's avatar
Julius Metz committed
641 642 643 644 645
        elif source_path.is_file() and source.endswith('.raw.gz'):
            source_paths.append(source_path)
    return source_paths


Julius Metz's avatar
Julius Metz committed
646
def data_from_file(arguments):
Julius Metz's avatar
Julius Metz committed
647 648 649 650 651 652 653 654 655 656
    """make data and filter infos of one collectl file

    Arguments:
        arguments {tuple} -- is a tuple of arguments for the function:
                    path {Path} -- collectldata file path
                    collectl {str} -- collectl command
                    shorten_cmds {bool} -- para for parse_file
                    coarsest {bool} -- para for parse_file
                    filtercmds {bool} -- if True cmds will be filter else not
                    filtervalue {int} -- para for filter
Matthias Lieber's avatar
Matthias Lieber committed
657
                    filtertype {str} -- which filter will be called
Julius Metz's avatar
Julius Metz committed
658
                    config {dict} -- plots and merge Config
Julius Metz's avatar
Julius Metz committed
659
    Returns:
Julius Metz's avatar
Julius Metz committed
660 661 662
        (str, dict, dict) -- 1. hostname
                             2. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                             3. filter_infos = {metrics: [cmds, ...]
Julius Metz's avatar
Julius Metz committed
663
    """
Julius Metz's avatar
Julius Metz committed
664
    path, collectl, shorten_cmds, coarsest, filtercmds, filtervalue, filtertype, config = arguments
Julius Metz's avatar
Julius Metz committed
665 666 667 668 669
    host = ''
    with gzip.open(path, 'r') as f:
        for line in f:
            if line.startswith(b'# Host:'):
                host = re.search(r'# Host: *([^ ]+)', line.decode()).group(1)
Julius Metz's avatar
Julius Metz committed
670
    data, filter_data = parse_file(path, collectl, shorten_cmds, coarsest, config)
Julius Metz's avatar
Julius Metz committed
671 672

    filter_infos = None
Julius Metz's avatar
Julius Metz committed
673 674
    if filtercmds:
        filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
Julius Metz's avatar
Julius Metz committed
675 676 677 678 679 680 681 682 683 684 685 686


    # make new dict with sorted cmd from big to small
    filter_infos_sorted = {}
    for metric in config['NEEDED_VALUES']:
        filter_infos_sorted[metric] = []
        # go over all sorted commands and add if is not filtered out
        for (cmd, _) in sorted(filter_data['commands'].items(), key=lambda x: x[1][metric], reverse=True):
            if not filtercmds or cmd in filter_infos[metric]:
                filter_infos_sorted[metric].append(cmd)

    return host, data, filter_infos_sorted
Julius Metz's avatar
Julius Metz committed
687

688

Julius Metz's avatar
Julius Metz committed
689
@click.command(help='Generate htmlfiles with Plotlyplots with data from collectlfiles(".raw.gz")')
690
@click.option('--source', '-s', multiple=True, default=['.'], show_default=True, type=click.Path(exists=True), help='source for the plots. (collectl data(.raw.gz) file or directory with collectl data) multiple useable')
Julius Metz's avatar
Julius Metz committed
691
@click.option('--collectl', '-c', default='collectl', show_default=True, help='collectl command')
Julius Metz's avatar
Julius Metz committed
692
@click.option('--plotlypath', '-p', type=click.Path(exists=True), help='path to plotly.js')
Julius Metz's avatar
Julius Metz committed
693
@click.option('--destination', '-d', default='.', type=click.Path(exists=True), show_default=True, help='path to directory where directory with plots will be created')
Julius Metz's avatar
Julius Metz committed
694
@click.option('--configpath', default=None, type=click.Path(exists=True), help='python file with plot and merge infos see doku for detail')
Julius Metz's avatar
Julius Metz committed
695
@click.option('--shorten/--notshorten', default=True, help='commands will be shorted only to name')
696
@click.option('--coarsest', is_flag=True, help='commands will be shorted only to type (bash, perl, ...)')
Julius Metz's avatar
Julius Metz committed
697
@click.option('--filtercmds/--notfiltercmds', default=True, help='filtering or not')
Matthias Lieber's avatar
Matthias Lieber committed
698
@click.option('--filtervalue', help='Parameter which is given to the filter.')
Julius Metz's avatar
Julius Metz committed
699 700 701 702 703
@click.option('--filtertype',
              type=click.Choice(FILTER_FUNCTIONS.keys(), case_sensitive=False),
              default=DEFAULT_FILTER, show_default=True,
              help='Filter which is to be used.',
              )
Julius Metz's avatar
Julius Metz committed
704 705 706
@click.option('--force', is_flag=True, help='override existing plot directory if exist')
def main(source, collectl, plotlypath, destination, configpath,
         shorten, coarsest, filtercmds, filtervalue, filtertype, force):
Julius Metz's avatar
Julius Metz committed
707
    source_paths = get_sources(source)
708

Julius Metz's avatar
Julius Metz committed
709
    if not source_paths:
Julius Metz's avatar
Julius Metz committed
710 711 712 713 714 715 716 717 718
        print('no valid source found')
        exit(1)

    if not Path(destination).is_dir():
        print('destination is no valid directory')
        exit(1)

    plots_dir = Path(destination, 'collectlplots')
    if plots_dir.is_dir():
Julius Metz's avatar
Julius Metz committed
719 720 721 722 723 724 725 726 727
        if force:
            shutil.rmtree(plots_dir)
        else:
            print('in destination "collectlplots" already exist')
            exit(1)

    config_module = default_plot_conf
    if configpath:
        if configpath.endswith('.py'):
728
            config_module = importlib.machinery.SourceFileLoader('config', configpath).load_module()
Julius Metz's avatar
Julius Metz committed
729 730 731 732 733 734
        else:
            print('given config isn`t a ".py" Python file')
            exit(1)

    # validate config and make dict from values
    config = {}
Julius Metz's avatar
Julius Metz committed
735
    for config_var_name, validator_dict in VALIDATIONS_INFOS.items():
Julius Metz's avatar
Julius Metz committed
736 737 738 739 740
        try:
            config[config_var_name] = getattr(config_module, config_var_name)
        except AttributeError:
            print('{} missing in config'.format(config_var_name))
            exit(1)
Julius Metz's avatar
Julius Metz committed
741 742 743 744
        valid, errormsg = check_valid(config[config_var_name], validator_dict)
        if not valid:
            print('In {0}: \n{1}'.format(config_var_name, errormsg))
            exit(1)
745

Julius Metz's avatar
Julius Metz committed
746 747 748
    data_colllect_functions = []
    for source_path in source_paths:
        data_colllect_functions.append((source_path,
Julius Metz's avatar
Julius Metz committed
749 750 751 752 753 754
                                        collectl,
                                        shorten,
                                        coarsest,
                                        filtercmds,
                                        filtervalue,
                                        FILTER_FUNCTIONS[filtertype],
Julius Metz's avatar
Julius Metz committed
755
                                        config,
Julius Metz's avatar
Julius Metz committed
756
                                        ))
757

Matthias Lieber's avatar
Matthias Lieber committed
758
    # use multiprocessing to parse all collectl output files independently from each other in parallel
Julius Metz's avatar
Julius Metz committed
759 760 761 762
    pool = mp.Pool(min(len(data_colllect_functions), mp.cpu_count()))
    results = pool.map(data_from_file, data_colllect_functions)
    pool.close()
    hosts_data = {}
Julius Metz's avatar
Julius Metz committed
763
    cmd_all = set()
Julius Metz's avatar
Julius Metz committed
764

Julius Metz's avatar
Julius Metz committed
765 766
    for host, data, filter_infos in results:
        hosts_data[host] = {'data': data, 'filter_infos': filter_infos}
Julius Metz's avatar
Julius Metz committed
767
        cmd_all.update([cmd for cmds in filter_infos.values() for cmd in cmds])
Julius Metz's avatar
Julius Metz committed
768

Julius Metz's avatar
Julius Metz committed
769 770
    cmd_colors = {cmd: config['PLOTLY_COLORS'][i % len(config['PLOTLY_COLORS'])] for i, cmd in enumerate(sorted(list(cmd_all)))}
    cmd_colors.update(config['PLOTLY_STATIC_COLORS'])
Julius Metz's avatar
Julius Metz committed
771

Matthias Lieber's avatar
Matthias Lieber committed
772
    # for each host and each plot (as given in config) call make_plot to create html div with plotly
Julius Metz's avatar
Julius Metz committed
773 774 775
    start_plots_build = time.time()
    plots_dict = {}
    for host, host_data in hosts_data.items():
Julius Metz's avatar
Julius Metz committed
776
        plots_dict[host] = {plot_config['name']: [] for plot_config in config['PLOTS_CONFIG']}
Julius Metz's avatar
Julius Metz committed
777
        for plot_config in config['PLOTS_CONFIG']:
Matthias Lieber's avatar
Matthias Lieber committed
778
            plots_dict[host][plot_config['name']].append( make_plot(
Julius Metz's avatar
Julius Metz committed
779 780 781 782 783 784
                host_data['data'],
                host_data['filter_infos'],
                cmd_colors,
                plot_config['plotly_settings'],
                {'host': host},
                plot_config['needed_key'],
785
                plot_config.get('relative-absolute_xaxis', False),
Julius Metz's avatar
Julius Metz committed
786 787 788
            ))
    print("plots build in {:.1f}s".format(time.time() - start_plots_build))

Julius Metz's avatar
Julius Metz committed
789

Matthias Lieber's avatar
Matthias Lieber committed
790
    # create output directory and copy plotly.js
Julius Metz's avatar
Julius Metz committed
791
    plots_dir.mkdir()
Julius Metz's avatar
Julius Metz committed
792 793
    if plotlypath is None:
        plotlypath = Path(__file__).with_name('plotly-latest.min.js')
Julius Metz's avatar
Julius Metz committed
794
    try:
Julius Metz's avatar
Julius Metz committed
795
        shutil.copy(plotlypath, str(Path(plots_dir, 'plotly.js')))
Julius Metz's avatar
Julius Metz committed
796 797 798
    except shutil.SameFileError:
        pass

Matthias Lieber's avatar
Matthias Lieber committed
799
    # for each host create html file with plots
Julius Metz's avatar
Julius Metz committed
800 801
    time_sites = time.time()
    for host, site_plots in plots_dict.items():
Julius Metz's avatar
Julius Metz committed
802 803
        with Path(plots_dir, host+'-plots.html').open(mode='w') as plots_file:
            plots_file.write(build_html(site_plots))
Julius Metz's avatar
Julius Metz committed
804
    print("write/build of websites took {:.1f}s".format(time.time() - time_sites))
Julius Metz's avatar
Julius Metz committed
805 806 807 808


if __name__ == '__main__':
    main()