Collectl2plotly.py 31.5 KB
Newer Older
1
import re
Julius Metz's avatar
Julius Metz committed
2
import gzip
Julius Metz's avatar
Julius Metz committed
3 4
import subprocess
from pathlib import Path
5 6
import copy
import time
7
import datetime
Julius Metz's avatar
Julius Metz committed
8 9
import multiprocessing as mp
import shutil
Julius Metz's avatar
Julius Metz committed
10
import importlib
Julius Metz's avatar
Julius Metz committed
11 12
import pickle
import json
Julius Metz's avatar
Julius Metz committed
13 14

import click
15 16
from yattag import Doc

Julius Metz's avatar
Julius Metz committed
17

18 19 20
import Collectl2plotly.filter_func as filter_func
import Collectl2plotly.value_merger as value_merger
import Collectl2plotly.Collectl2plotly_Plots_Config as default_plot_conf
Julius Metz's avatar
Julius Metz committed
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36


### Filter Configs

# registered filter functions
# filter functions must be in filter_func.py
# key  : Display name
# value: function name
FILTER_FUNCTIONS = {'hardvalue': 'filter_hardvalue', 'average': 'filter_average'}

# Default filter function if no explicit is given
# must be a display name of a function
DEFAULT_FILTER = 'average'

### validation Config

Julius Metz's avatar
Julius Metz committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
# Merger that exist and posible kwargs/parameter
MERGER = {
    'x2oneaddition_int': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
    },
    'x2oneaddition_float': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
    }
}

# validation infos of config
# key = config variablename to validate
# value = validator dict or type/tuple of types
# hints!!!!
# '__merger__' can be used as type for the merger tuple.
# in fixed_keys and 'changing_keys' the 'validator' need not be given if sub-elements are not to be validated.
#
# validator dict has follow structur:
# 'type' : required, is the possible type/s of the obj that are valid
# 'emptyable': list/dict/tuple can be empty if emptyable is True. Default: False
# 'subvalue_validator': is a validatordict or type/tuple of types to check all elems are valid if type is list/tuple
# 'fixed_keys': only if type is dict. takes a dict:
#               key: a known key of the current dict that is checked
#               value: dict with following possible keys:
#                     'required': if False the key must not in the dict Default: True
#                     'validator':  is a validatordict or type/tuple of types to validate the value
# 'changing_keys': only if type is dict. takes a dict:
#               'key_type': type of the keys in the dict current dict
#               'validator':  is a validatordict or type/tuple of types to validate the values of the dict
#
VALIDATIONS_INFOS = {
    'NEEDED_VALUES_DEFAULTS': {
        'type': dict,
        'fixed_keys': {
            'default_base_value': {},
            'default_merger': {'validator': '__merger__'},
        },
    },
    'NEEDED_VALUES': {
        'type': dict,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': dict,
                'fixed_keys': {
                    'keys': {'validator': {'type': list, 'subvalue_validator': str}},
                    'merger': {'validator': '__merger__'},
                    'base_value': {'required': False}
                },
            },
        },
    },
    'PLOTS_CONFIG': {
        'type': list,
        'subvalue_validator': {
            'type': dict,
            'fixed_keys': {
                'name': {'validator': str},
                'needed_key': {'validator': str},
                'relative-absolute_xaxis': {'required': False, 'validator': bool},
                'plotly_settings': {'validator': dict},
            },
        },
    },
    'NAME_SPEZIAL_PARAMETER': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': list,
                'emptyable':True,
                'subvalue_validator': str,
            },
        },
    },
    'COMAND_BLACKLIST': {
        'type': list,
        'emptyable': True,
        'subvalue_validator': str,
    },
    'PLOTLY_COLORS': {
        'type': list,
        'subvalue_validator': str,
    },
    'PLOTLY_STATIC_COLORS': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': str,
        },
    },
}

Julius Metz's avatar
WC  
Julius Metz committed
136

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
### Other

# config for find best axis interval and unit
# from smallest unit to greatest
# intervals = possible intervals of the axis(needed)
# max = conversion value if not given greatest supported unit
# unit = str with unit name
AXIS_INTERVAL_REL = [
    {
        'max': 60,
        'intervals': [1, 5, 10],
        'unit': 'sec',
    },
    {
        'max': 60,
        'intervals': [0.5, 1, 5, 10],
        'unit': 'min',
    },
    {
        'intervals': [0.25 ,0.5, 1, 5, 10],
        'unit': 'h',
    }
]

Julius Metz's avatar
Julius Metz committed
161 162 163
#pattern for split of commands (with \" and \ )
FIELDS_PATTERN = re.compile(r'(?:(?:\s*[^\\]|\A)\"(.*?[^\\]|)\"|(?:\s+|\A)(?=[^\s])(.*?[^\\])(?= |\Z))')
#(?:\"(.*?)\"|(\S+))
164 165


Julius Metz's avatar
Julius Metz committed
166 167 168 169 170 171 172 173 174 175

def is_merger(to_check):
    """Check is given obj a valid merger. Possible merger are given in MERGER.

    Arguments:
        to_check {unkown} -- obj to test

    Returns:
        (bool, str) -- return true if it is a merger false and error if not
    """
176 177

    # check is tuple with 2 elems
Julius Metz's avatar
Julius Metz committed
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
    try:
        name, mergerkwargs = to_check
    except ValueError:
        return False, "'{0}' has no right merger structur expected: (name[str], kwargs[dict])".format(to_check)

    if isinstance(name, str) and isinstance(mergerkwargs, dict):
        if name in MERGER:
            for kwarg_name, value in mergerkwargs.items():
                if kwarg_name in MERGER[name]:
                    if 'choices' in MERGER[name][kwarg_name]:
                        if not value in MERGER[name][kwarg_name]['choices']:
                            return False, '{0} {1} expect one of {2} got: {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['choices'], value
                            )
                    else:
                        if not isinstance(value, MERGER[name][kwarg_name]['type']):
                            return False, '{1} of {0} expect {2} not {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['type'], type(value)
                            )
                else:
                    return False, '{1} is unknown parameter of {0}'.format(name, kwarg_name)
        else:
            return False, '"{0}" is unknown merger'.format(name)
    else:
        return False, '"{0}" has no right merger structur expected: (name[str], kwargs[dict])'.format(to_check)

    return True, None


def check_valid(to_check, validationinfos):
    """check object is valid from the perspective of validationinfos

    Arguments:
        to_check {unkown} -- object to check is valid
        validationinfos {dict or type or tuple with types} -- infos what is  needed that to_check is valid

    Returns:
        (bool, str) -- return true if it is valid, false and error if not
    """
    if isinstance(validationinfos, dict):
        if validationinfos['type'] == '__merger__':
            return is_merger(to_check)
        else:
            if isinstance(to_check, validationinfos['type']):
222
                if type(to_check) == dict:
Julius Metz's avatar
Julius Metz committed
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

                    for fixed_name, infos in validationinfos.get('fixed_keys', {}).items():
                        if fixed_name in to_check:
                            if 'validator' in infos:
                                valid, error_msg = check_valid(to_check[fixed_name], infos['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(fixed_name, error_msg)

                        elif infos.get('required', True):
                            return False, '{0} missing key: "{1}"'.format(to_check, fixed_name)

                    if 'changing_keys' in validationinfos:
                        for key, subvalue in to_check.items():
                            if not isinstance(key, validationinfos['changing_keys']['key_type']):
                                return False, '{0} key has wrong type expected {1} got {2}'.format(
                                    key,  validationinfos['changing_keys']['key_type'], type(key),
                                )
                            if 'validator' in validationinfos['changing_keys']:
                                valid, error_msg =  check_valid(subvalue, validationinfos['changing_keys']['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(key, error_msg)

247
                if isinstance(to_check, (list, tuple)):
Julius Metz's avatar
Julius Metz committed
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

                    if 'subvalue_validator' in validationinfos:
                        for i, subvalue in enumerate(to_check):
                            valid, error_msg = check_valid(subvalue, validationinfos['subvalue_validator'])
                            if not valid:
                                return valid, '[{0}] => {1}'.format(i, error_msg)

            else:
                return False, '{0} is wrong type expected {1} got {2}'.format(
                        to_check, validationinfos['type'], type(to_check)
                    )

    else:
        if validationinfos == '__merger__':
            return is_merger(to_check)
        else:
            if not isinstance(to_check, validationinfos):
                return False, '{0} is {1} expected: {2}'.format(
                    to_check, type(to_check), validationinfos,
                )

    return True, None


274
def datestr2date(datestr):
Julius Metz's avatar
Julius Metz committed
275 276 277 278 279 280 281 282
    """Converts a "datestring" to a date Object.

    Arguments:
        datestr {str} -- string of a Date example: 20191224

    Returns:
        datetime.date -- date of the string
    """
283
    return datetime.date(
Julius Metz's avatar
Julius Metz committed
284 285 286 287
        int(datestr[:4]),
        int(datestr[4:6]),
        int(datestr[6:8]),
    )
288 289


Julius Metz's avatar
Julius Metz committed
290
def get_cmdname(cmd, spezial_parameters_of_all, coarsest=False):
291 292 293 294 295 296 297 298 299 300 301
    """search in complete commandstring the name of the skript or the command that is used

    Arguments:
        cmd {str} -- complete commandstring

    Keyword Arguments:
        coarsest {bool} -- return only the call function(example: bash, python) if True (default: {False})

    Returns:
        str -- new cmd name
    """
Matthias Lieber's avatar
Matthias Lieber committed
302
    # search for (for example) bash_function="python" in cmd="/usr/bin/python3 my_script.py"
Julius Metz's avatar
Julius Metz committed
303 304 305
    # split command with regex
    cmd_splited = [x[0] or x[1] for x in FIELDS_PATTERN.findall(cmd)]
    #shlex.split(cmd)
306 307
    bash_function = cmd_splited[0].split('/')[-1]
    bash_function = re.search(r'[^\W\n]+', bash_function).group(0)
Matthias Lieber's avatar
Matthias Lieber committed
308
    # check if bash_function is known, if not, return bash_function
Julius Metz's avatar
Julius Metz committed
309 310
    spezial_parameters = spezial_parameters_of_all.get(bash_function, None)
    if coarsest or spezial_parameters is None:
311 312
        return bash_function
    skip = False
Matthias Lieber's avatar
Matthias Lieber committed
313
    # search script/program name within the parameters and only return this without path
314 315 316 317
    for position, parameter in enumerate(cmd_splited[1:]):
        if skip:
            skip = False
            continue
Julius Metz's avatar
Julius Metz committed
318
        if parameter in spezial_parameters:
319 320 321 322
            skip = True
            continue
        if bash_function == 'bash' or bash_function == 'sh' and parameter == '-c':
            return bash_function + ' -c'
Julius Metz's avatar
Julius Metz committed
323
            # return cmd_splited[position+1]
324 325 326 327 328 329
        if parameter.startswith('-'):
            continue
        return parameter.split('/')[-1]
    return cmd


Julius Metz's avatar
Julius Metz committed
330
def parse_file(path, collectl, shorten_cmds, coarsest, config):
Julius Metz's avatar
Julius Metz committed
331 332 333 334 335 336 337 338
    """start subproccess collectl than parse the output and merge.
       After that build usefull dict from parsed data

    Arguments:
        path {Path} -- path to collectl file to parse
        collectl {str} -- collectl command
        shorten_cmds {bool} -- if True cmd will be shorted by get_cmdname
        coarsest {bool} -- parameter of get_cmdname
Julius Metz's avatar
Julius Metz committed
339 340
        config {dict} -- plots and merge Config

Julius Metz's avatar
Julius Metz committed
341 342

    Returns:
Julius Metz's avatar
Julius Metz committed
343 344 345
        (dict, dict) --  1. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                         2. data for filter function :
                         {'number_of_values': X, 'commands':{ cmd:{'number_of_values': X, metrics:SUM, ...}}, metrics:SUM, ...}
Julius Metz's avatar
Julius Metz committed
346
    """
347
    collectl_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
348
    # run collectl in playback mode and read output into list
349
    process = subprocess.run(
Matthias Lieber's avatar
Matthias Lieber committed
350
        [collectl, '-P', '-p', path, '-sZ'], stdout=subprocess.PIPE, check=True,
351
    )
Julius Metz's avatar
Julius Metz committed
352 353
    print('collectl make table took {:.1f}s'.format(
        time.time() - collectl_starttime))
354
    parsing_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
355
    # output contains all data!
Julius Metz's avatar
Julius Metz committed
356
    output = process.stdout.decode().splitlines()
Matthias Lieber's avatar
Matthias Lieber committed
357
    # get table head
Julius Metz's avatar
Julius Metz committed
358
    head = output.pop(0).split(' ')
359 360 361 362 363 364 365
    for possible_head in output[:]:
        if possible_head.startswith('#'):
            head = possible_head.split(' ')
            output.remove(possible_head)
        else:
            break

Matthias Lieber's avatar
Matthias Lieber committed
366
    # get template of an entry from the head
Julius Metz's avatar
Julius Metz committed
367
    head[0] = head[0][1:]
Julius Metz's avatar
Julius Metz committed
368 369
    head_indexes_dict = {
        head_title: index for index, head_title in enumerate(head)}
370 371
    empty_dict_with_value_titles = {
        value_title: copy.deepcopy(
Julius Metz's avatar
Julius Metz committed
372
            value_title_settings.get(
Julius Metz's avatar
Julius Metz committed
373
                'base_value', config['NEEDED_VALUES_DEFAULTS']['default_base_value']
Julius Metz's avatar
Julius Metz committed
374
            )
Julius Metz's avatar
Julius Metz committed
375
        ) for value_title, value_title_settings in config['NEEDED_VALUES'].items()
376
    }
Julius Metz's avatar
Julius Metz committed
377

Matthias Lieber's avatar
Matthias Lieber committed
378 379
    # parse all output lines
    entrys_data = {}
Julius Metz's avatar
WC  
Julius Metz committed
380
    cmd_cmdshort_dict = {}
Julius Metz's avatar
Julius Metz committed
381 382
    merger_lookup_dict = {}

Julius Metz's avatar
Julius Metz committed
383
    for entry in output:
Matthias Lieber's avatar
Matthias Lieber committed
384
        # split by ' ' (exclude command from splitting)
385
        splited_entry = entry.split(' ', len(head_indexes_dict)-1)
Matthias Lieber's avatar
Matthias Lieber committed
386
        # get command string and shorten
Julius Metz's avatar
Julius Metz committed
387
        cmd = splited_entry[-1]
Julius Metz's avatar
Julius Metz committed
388
        if shorten_cmds or coarsest:
Julius Metz's avatar
WC  
Julius Metz committed
389 390 391
            if cmd in cmd_cmdshort_dict:
                cmd = cmd_cmdshort_dict[cmd]
            else:
Julius Metz's avatar
Julius Metz committed
392
                short_cmd = get_cmdname(cmd, config['NAME_SPEZIAL_PARAMETER'], coarsest=coarsest)
Julius Metz's avatar
WC  
Julius Metz committed
393 394
                cmd_cmdshort_dict[cmd]= short_cmd
                cmd = short_cmd
Julius Metz's avatar
Julius Metz committed
395
        if cmd in config['COMAND_BLACKLIST']:
Julius Metz's avatar
Julius Metz committed
396
            continue
Matthias Lieber's avatar
Matthias Lieber committed
397
        # create dict for each command
Julius Metz's avatar
Julius Metz committed
398 399
        if not cmd in entrys_data:
            entrys_data[cmd] = {}
Matthias Lieber's avatar
Matthias Lieber committed
400
        # get datetime obj for current entry
Julius Metz's avatar
Julius Metz committed
401 402
        tmp_datetime = datetime.datetime.combine(
            datestr2date(splited_entry[head_indexes_dict['Date']]),
Matthias Lieber's avatar
Matthias Lieber committed
403
            datetime.time(*[int(n) for n in splited_entry[head_indexes_dict['Time']].split(':')]),
Julius Metz's avatar
Julius Metz committed
404
        )
Matthias Lieber's avatar
Matthias Lieber committed
405
        # if datetime not yet existing, add new entry from template
Julius Metz's avatar
Julius Metz committed
406
        if not tmp_datetime in entrys_data[cmd]:
Julius Metz's avatar
Julius Metz committed
407 408
            entrys_data[cmd][tmp_datetime] = pickle.loads(pickle.dumps(
                empty_dict_with_value_titles, -1))
Matthias Lieber's avatar
Matthias Lieber committed
409 410
        # get values from data as given in NEEDED_VALUES and run specified merger function
        # to merge multiple values with same timestamp or rescale (e.g. to GB)
Julius Metz's avatar
Julius Metz committed
411 412 413 414 415
        for value_title, value_title_settings in config['NEEDED_VALUES'].items():
            merger, merger_kwargs = value_title_settings.get('merger', config['NEEDED_VALUES_DEFAULTS']['default_merger'])
            if not merger in merger_lookup_dict:
                merger_lookup_dict[merger] = getattr(value_merger, merger)
            entrys_data[cmd][tmp_datetime][value_title] = merger_lookup_dict[merger](
Julius Metz's avatar
Julius Metz committed
416 417
                entrys_data[cmd][tmp_datetime][value_title],
                *[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
Julius Metz's avatar
Julius Metz committed
418
                **merger_kwargs
419
            )
Julius Metz's avatar
Julius Metz committed
420
    print('parsing/merge took {:.1f}s'.format(time.time() - parsing_starttime))
421
    dictbuild_starttime = time.time()
422

Matthias Lieber's avatar
Matthias Lieber committed
423 424
    # create lists entry_data_plotfriendly[cmd][metric] = [ values, ... ] with the actual data to plot
    # and sum up all metrics for each command to enable filtering of non-interesting commands later on
425
    entry_data_plotfriendly = {}
Julius Metz's avatar
Julius Metz committed
426
    plot_filter_data = pickle.loads(pickle.dumps(empty_dict_with_value_titles, -1))
427 428 429
    plot_filter_data['number_of_values'] = 0
    plot_filter_data['commands'] = {}
    for cmd, cmd_data in entrys_data.items():
Julius Metz's avatar
Julius Metz committed
430 431
        plot_filter_data['commands'][cmd] = pickle.loads(pickle.dumps(
            empty_dict_with_value_titles, -1))
432
        plot_filter_data['commands'][cmd]['number_of_values'] = 0
Julius Metz's avatar
Julius Metz committed
433
        entry_data_plotfriendly[cmd] = {key: [] for key in config['NEEDED_VALUES']}
434
        entry_data_plotfriendly[cmd]['datetime'] = []
Julius Metz's avatar
Julius Metz committed
435
        for cmd_data_time, cmd_data_values in cmd_data.items():
436 437 438 439 440 441 442 443
            entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
            for cmd_data_key, cmd_data_value in cmd_data_values.items():
                entry_data_plotfriendly[cmd][cmd_data_key].append(cmd_data_value)

                plot_filter_data['commands'][cmd][cmd_data_key] += cmd_data_value
                plot_filter_data['commands'][cmd]['number_of_values'] += 1
                plot_filter_data[cmd_data_key] += cmd_data_value
                plot_filter_data['number_of_values'] += 1
Julius Metz's avatar
Julius Metz committed
444 445 446 447
    print(
        'data dict/ filter_data_dict build took {:.1f}s'.format(
            time.time() - dictbuild_starttime),
    )
448
    return entry_data_plotfriendly, plot_filter_data
Julius Metz's avatar
Julius Metz committed
449

Julius Metz's avatar
Julius Metz committed
450

451 452 453 454 455 456 457 458 459 460 461 462
def make_relative_xaxi(all_values):
    """build lists with relative values for xaxis

    Arguments:
        all_values {[datetime, ..]} -- list of all datetime objects for plot

    Returns:
        (list, list) -- 1.: list with the relative values that are displayed on the xaxis
                        2.: list with the datetime as str of the absolute values of xaxis
    """
    min_value = min(all_values)
    max_value = max(all_values)
463 464

    # Is the reference value where the next possible value is searched. For a good number of ticks on the axis.
465
    guideline_tickcounts = 10
466

467 468 469 470
    end_value = (max_value - min_value).total_seconds()
    conversion_factor = 1

    for current_unit in AXIS_INTERVAL_REL:
471
        #  Convert to higher units until the correct one is sufficient
472 473 474 475 476
        if 'max' in current_unit and end_value > current_unit['max']:
            end_value /= current_unit['max']
            conversion_factor *= current_unit['max']
            continue

477
        # find out the best possible interval
478 479 480 481 482
        distances = [
            max(end_value // interval - guideline_tickcounts, guideline_tickcounts - end_value // interval) \
                for interval in current_unit['intervals']
        ]
        interval = current_unit['intervals'][distances.index(min(distances))]
483 484

        # make array with plotly axis data
485 486 487 488 489 490 491 492
        xaxis_ticks = [interval * i for i in range(int(end_value/interval)+1)]
        if len(xaxis_ticks) == 1:
            xaxis_ticks.append(1)
        xaxis_ticks_values = [str(min_value + datetime.timedelta(seconds=tick * conversion_factor)) for tick in xaxis_ticks]
        return xaxis_ticks, xaxis_ticks_values, current_unit['unit']


def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key, relative_xaxis):
Julius Metz's avatar
WC  
Julius Metz committed
493
    """Build a list of dicts in Plotlyconf style for the diffrent traces with the data from cmds_data of needed_key
494
        than make plot dict in Plotly style and change '{host}' in titels.
Julius Metz's avatar
Julius Metz committed
495 496 497 498 499

    Arguments:
        cmds_data {dict} -- is the data for the plot example: {'Command': {'datetime':[...], 'cpu': [...]}}
        filter_info {dict} -- has the information which cmds will be shown example: {'cpu': [cmds]}
        cmd_color {dict} -- assign a fixed plot color to each cmd example:  {'Command': 'rgb(0, 0, 128)'}
Julius Metz's avatar
WC  
Julius Metz committed
500 501
        plot_settings {dict} -- settings for Plotly
        plotly_format_vars {dict} -- values for Plotly settings
Julius Metz's avatar
Julius Metz committed
502
        needed_key {str} -- key of cmds_data for the values to be use
503
        relative_xaxis {bool} -- if true add buttons to change xaxis to relative
Julius Metz's avatar
Julius Metz committed
504 505

    Returns:
506
        dict -- with plotly jsons
Julius Metz's avatar
Julius Metz committed
507 508 509
    """
    plot_data = []
    for cmd in filter_info[needed_key]:
Julius Metz's avatar
Julius Metz committed
510 511 512
        plot_data.append({
            'x': [str(date) for date in cmds_data[cmd]['datetime']],
            'y': cmds_data[cmd][needed_key],
Julius Metz's avatar
Julius Metz committed
513
            'name': cmd,
Julius Metz's avatar
Julius Metz committed
514
            'marker': {
Julius Metz's avatar
Julius Metz committed
515 516
                'color': cmd_color[cmd],
            },
Julius Metz's avatar
Julius Metz committed
517
            **plot_settings['data'],
Julius Metz's avatar
WC  
Julius Metz committed
518 519 520 521
        }
        )


522 523 524
    layout = pickle.loads(pickle.dumps(plot_settings.get('layout', {}), -1))

    # replace {host} in titles
Julius Metz's avatar
Julius Metz committed
525 526 527
    if 'title' in layout:
        if type(layout['title']) == str:
            layout['title'] = layout['title'].format(
Julius Metz's avatar
Julius Metz committed
528 529 530
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
531
            layout['title']['text'] = layout['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
532 533
                **plotly_format_vars,
            )
Julius Metz's avatar
WC  
Julius Metz committed
534

Julius Metz's avatar
Julius Metz committed
535 536 537
    if 'yaxis' in layout and 'title' in layout['yaxis']:
        if type(layout['yaxis']['title']) == str:
            layout['yaxis']['title'] = layout['yaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
538 539 540
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
541
            layout['yaxis']['title']['text'] = layout['yaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
542 543
                **plotly_format_vars,
            )
544
    xtitle = ''
Julius Metz's avatar
Julius Metz committed
545 546 547
    if 'xaxis' in layout and 'title' in layout['xaxis']:
        if type(layout['xaxis']['title']) == str:
            layout['xaxis']['title'] = layout['xaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
548 549
                **plotly_format_vars,
            )
550
            xtitle = layout['xaxis']['title']
Julius Metz's avatar
Julius Metz committed
551
        else:
Julius Metz's avatar
Julius Metz committed
552
            layout['xaxis']['title']['text'] = layout['xaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
553 554
                **plotly_format_vars,
            )
555
            xtitle = layout['xaxis']['title']['text']
Julius Metz's avatar
WC  
Julius Metz committed
556

557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
    # add buttons for switching xaxis if 'relative-absolute_xaxis' in PLOT_CONF is set.
    if relative_xaxis:
            ticks_name, ticks_value, unit = make_relative_xaxi(
                [date for cmd in filter_info[needed_key] for date in cmds_data[cmd]['datetime']],
            )
            if not 'xaxis' in layout:
                layout['xaxis'] = {}
            if not 'updatemenus' in layout:
                layout['updatemenus'] = []
            layout['updatemenus'].append({
                'type': 'buttons',
                'x': 0.6,
                'y': 1.15,
                'direction': 'left',
                'buttons': [
                    {'args':[{'xaxis':{'title': xtitle, 'tickvals': None}}],
                    'label':'absolute xaxis',
                    'method':'relayout'},
                    {'args':[{'xaxis': {'title': 'runtime in {}'.format(unit), 'tickvals': ticks_value, 'ticktext': ticks_name}}],
                    'label':'relative xaxis',
                    'method':'relayout'},
                ],
                'showactive':True,
            })

    return {'data': json.dumps(plot_data), 'layout': json.dumps(layout), 'config': json.dumps(plot_settings.get('config', {}))}
Julius Metz's avatar
Julius Metz committed
583 584 585


def build_html(plots_dict):
Julius Metz's avatar
Julius Metz committed
586 587 588 589 590 591 592 593
    """build html site with the plots in plotsdict

    Arguments:
        plots_dict {dict} -- key: name of plot value: [plot, ...]

    Returns:
        str -- html website
    """
594 595 596 597
    doc, tag, text = Doc().tagtext()
    doc.asis('<!DOCTYPE html>')
    with tag('html'):
        with tag('head'):
Julius Metz's avatar
Julius Metz committed
598 599
            with tag('script', src='plotly.js'):
                pass
Julius Metz's avatar
Julius Metz committed
600 601 602 603
            with tag('script'):
                doc.asis("document.onreadystatechange = () => {if (document.readyState === 'complete') {")
                for name, plots in plots_dict.items():
                    for i, plot in enumerate(plots):
604 605
                        doc.asis("Plotly.newPlot(document.getElementById('{name}-plot-{number}'), JSON.parse('{data}'), JSON.parse('{layout}'), JSON.parse('{config}'));"\
                            .format(name=name, number=i, data=plot['data'], layout=plot['layout'], config=plot['config'])
Julius Metz's avatar
Julius Metz committed
606 607
                        )
                doc.asis('}}')
608 609 610 611 612
        with tag('body'):
            with tag('div', id='index'):
                with tag('h2'):
                    text('Index:')
                with tag('ul'):
Julius Metz's avatar
Julius Metz committed
613
                    for name in plots_dict:
614 615 616 617 618
                        with tag('li'):
                            with tag('a', href='#'+name):
                                text(name)
            for name, plots in plots_dict.items():
                with tag('div', id=name):
Julius Metz's avatar
Julius Metz committed
619 620 621
                    for i in range(len(plots)):
                        with tag('div', id='{}-plot-{}'.format(name, i)):
                            pass
622 623 624 625

    return doc.getvalue()


Julius Metz's avatar
Julius Metz committed
626 627 628 629 630 631 632 633 634 635 636 637 638
def get_sources(sources):
    """collect all sources (.raw.gz files)

    Arguments:
        sources {tuple} -- tuple of files/directorys

    Returns:
        list -- list of all .raw.gz files which was found
    """
    source_paths = []
    for source in sources:
        source_path =  Path(source)
        if source_path.is_dir():
Julius Metz's avatar
Julius Metz committed
639
            source_paths.extend(source_path.glob('*.raw.gz'))
Julius Metz's avatar
Julius Metz committed
640 641 642 643 644
        elif source_path.is_file() and source.endswith('.raw.gz'):
            source_paths.append(source_path)
    return source_paths


Julius Metz's avatar
Julius Metz committed
645
def data_from_file(arguments):
Julius Metz's avatar
Julius Metz committed
646 647 648 649 650 651 652 653 654 655
    """make data and filter infos of one collectl file

    Arguments:
        arguments {tuple} -- is a tuple of arguments for the function:
                    path {Path} -- collectldata file path
                    collectl {str} -- collectl command
                    shorten_cmds {bool} -- para for parse_file
                    coarsest {bool} -- para for parse_file
                    filtercmds {bool} -- if True cmds will be filter else not
                    filtervalue {int} -- para for filter
Matthias Lieber's avatar
Matthias Lieber committed
656
                    filtertype {str} -- which filter will be called
Julius Metz's avatar
Julius Metz committed
657
                    config {dict} -- plots and merge Config
Julius Metz's avatar
Julius Metz committed
658
    Returns:
Julius Metz's avatar
Julius Metz committed
659 660 661
        (str, dict, dict) -- 1. hostname
                             2. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                             3. filter_infos = {metrics: [cmds, ...]
Julius Metz's avatar
Julius Metz committed
662
    """
Julius Metz's avatar
Julius Metz committed
663
    path, collectl, shorten_cmds, coarsest, filtercmds, filtervalue, filtertype, config = arguments
Julius Metz's avatar
Julius Metz committed
664 665 666 667 668
    host = ''
    with gzip.open(path, 'r') as f:
        for line in f:
            if line.startswith(b'# Host:'):
                host = re.search(r'# Host: *([^ ]+)', line.decode()).group(1)
Julius Metz's avatar
Julius Metz committed
669
    data, filter_data = parse_file(path, collectl, shorten_cmds, coarsest, config)
Julius Metz's avatar
Julius Metz committed
670 671 672
    if filtercmds:
        filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
    else:
Julius Metz's avatar
Julius Metz committed
673
        filter_infos = {key: list(data.keys()) for key in config['NEEDED_VALUES']}
Julius Metz's avatar
Julius Metz committed
674 675
    return host, data, filter_infos

676

Julius Metz's avatar
Julius Metz committed
677
@click.command(help='Generate htmlfiles with Plotlyplots with data from collectlfiles(".raw.gz")')
678
@click.option('--source', '-s', multiple=True, default=['.'], show_default=True, type=click.Path(exists=True), help='source for the plots. (collectl data(.raw.gz) file or directory with collectl data) multiple useable')
Julius Metz's avatar
Julius Metz committed
679
@click.option('--collectl', '-c', default='collectl', show_default=True, help='collectl command')
Julius Metz's avatar
Julius Metz committed
680
@click.option('--plotlypath', '-p', type=click.Path(exists=True), help='path to plotly.js')
Julius Metz's avatar
Julius Metz committed
681
@click.option('--destination', '-d', default='.', type=click.Path(exists=True), show_default=True, help='path to directory where directory with plots will be created')
Julius Metz's avatar
Julius Metz committed
682
@click.option('--configpath', default=None, type=click.Path(exists=True), help='python file with plot and merge infos see doku for detail')
Julius Metz's avatar
Julius Metz committed
683
@click.option('--shorten/--notshorten', default=True, help='commands will be shorted only to name')
684
@click.option('--coarsest', is_flag=True, help='commands will be shorted only to type (bash, perl, ...)')
Julius Metz's avatar
Julius Metz committed
685
@click.option('--filtercmds/--notfiltercmds', default=True, help='filtering or not')
Matthias Lieber's avatar
Matthias Lieber committed
686
@click.option('--filtervalue', help='Parameter which is given to the filter.')
Julius Metz's avatar
Julius Metz committed
687 688 689 690 691
@click.option('--filtertype',
              type=click.Choice(FILTER_FUNCTIONS.keys(), case_sensitive=False),
              default=DEFAULT_FILTER, show_default=True,
              help='Filter which is to be used.',
              )
Julius Metz's avatar
Julius Metz committed
692 693 694
@click.option('--force', is_flag=True, help='override existing plot directory if exist')
def main(source, collectl, plotlypath, destination, configpath,
         shorten, coarsest, filtercmds, filtervalue, filtertype, force):
Julius Metz's avatar
Julius Metz committed
695
    source_paths = get_sources(source)
696

Julius Metz's avatar
Julius Metz committed
697
    if not source_paths:
Julius Metz's avatar
Julius Metz committed
698 699 700 701 702 703 704 705 706
        print('no valid source found')
        exit(1)

    if not Path(destination).is_dir():
        print('destination is no valid directory')
        exit(1)

    plots_dir = Path(destination, 'collectlplots')
    if plots_dir.is_dir():
Julius Metz's avatar
Julius Metz committed
707 708 709 710 711 712 713 714 715
        if force:
            shutil.rmtree(plots_dir)
        else:
            print('in destination "collectlplots" already exist')
            exit(1)

    config_module = default_plot_conf
    if configpath:
        if configpath.endswith('.py'):
716
            config_module = importlib.machinery.SourceFileLoader('config', configpath).load_module()
Julius Metz's avatar
Julius Metz committed
717 718 719 720 721 722
        else:
            print('given config isn`t a ".py" Python file')
            exit(1)

    # validate config and make dict from values
    config = {}
Julius Metz's avatar
Julius Metz committed
723 724
    a = time.time()
    for config_var_name, validator_dict in VALIDATIONS_INFOS.items():
Julius Metz's avatar
Julius Metz committed
725 726 727 728 729
        try:
            config[config_var_name] = getattr(config_module, config_var_name)
        except AttributeError:
            print('{} missing in config'.format(config_var_name))
            exit(1)
Julius Metz's avatar
Julius Metz committed
730 731 732 733
        valid, errormsg = check_valid(config[config_var_name], validator_dict)
        if not valid:
            print('In {0}: \n{1}'.format(config_var_name, errormsg))
            exit(1)
734

Julius Metz's avatar
Julius Metz committed
735 736 737
    data_colllect_functions = []
    for source_path in source_paths:
        data_colllect_functions.append((source_path,
Julius Metz's avatar
Julius Metz committed
738 739 740 741 742 743
                                        collectl,
                                        shorten,
                                        coarsest,
                                        filtercmds,
                                        filtervalue,
                                        FILTER_FUNCTIONS[filtertype],
Julius Metz's avatar
Julius Metz committed
744
                                        config,
Julius Metz's avatar
Julius Metz committed
745
                                        ))
746

Matthias Lieber's avatar
Matthias Lieber committed
747
    # use multiprocessing to parse all collectl output files independently from each other in parallel
Julius Metz's avatar
Julius Metz committed
748 749 750 751
    pool = mp.Pool(min(len(data_colllect_functions), mp.cpu_count()))
    results = pool.map(data_from_file, data_colllect_functions)
    pool.close()
    hosts_data = {}
Julius Metz's avatar
Julius Metz committed
752
    cmd_all = set()
Julius Metz's avatar
Julius Metz committed
753

Julius Metz's avatar
Julius Metz committed
754 755
    for host, data, filter_infos in results:
        hosts_data[host] = {'data': data, 'filter_infos': filter_infos}
Julius Metz's avatar
Julius Metz committed
756
        cmd_all.update([cmd for cmds in filter_infos.values() for cmd in cmds])
Julius Metz's avatar
Julius Metz committed
757

Julius Metz's avatar
Julius Metz committed
758 759
    cmd_colors = {cmd: config['PLOTLY_COLORS'][i % len(config['PLOTLY_COLORS'])] for i, cmd in enumerate(sorted(list(cmd_all)))}
    cmd_colors.update(config['PLOTLY_STATIC_COLORS'])
Julius Metz's avatar
Julius Metz committed
760

Matthias Lieber's avatar
Matthias Lieber committed
761
    # for each host and each plot (as given in config) call make_plot to create html div with plotly
Julius Metz's avatar
Julius Metz committed
762 763 764
    start_plots_build = time.time()
    plots_dict = {}
    for host, host_data in hosts_data.items():
Julius Metz's avatar
Julius Metz committed
765
        plots_dict[host] = {plot_config['name']: [] for plot_config in config['PLOTS_CONFIG']}
Julius Metz's avatar
Julius Metz committed
766
        for plot_config in config['PLOTS_CONFIG']:
Matthias Lieber's avatar
Matthias Lieber committed
767
            plots_dict[host][plot_config['name']].append( make_plot(
Julius Metz's avatar
Julius Metz committed
768 769 770 771 772 773
                host_data['data'],
                host_data['filter_infos'],
                cmd_colors,
                plot_config['plotly_settings'],
                {'host': host},
                plot_config['needed_key'],
774
                plot_config.get('relative-absolute_xaxis', False),
Julius Metz's avatar
Julius Metz committed
775 776 777
            ))
    print("plots build in {:.1f}s".format(time.time() - start_plots_build))

Julius Metz's avatar
Julius Metz committed
778

Matthias Lieber's avatar
Matthias Lieber committed
779
    # create output directory and copy plotly.js
Julius Metz's avatar
Julius Metz committed
780
    plots_dir.mkdir()
Julius Metz's avatar
Julius Metz committed
781 782
    if plotlypath is None:
        plotlypath = Path(__file__).with_name('plotly-latest.min.js')
Julius Metz's avatar
Julius Metz committed
783
    try:
Julius Metz's avatar
Julius Metz committed
784
        shutil.copy(plotlypath, str(Path(plots_dir, 'plotly.js')))
Julius Metz's avatar
Julius Metz committed
785 786 787
    except shutil.SameFileError:
        pass

Matthias Lieber's avatar
Matthias Lieber committed
788
    # for each host create html file with plots
Julius Metz's avatar
Julius Metz committed
789 790
    time_sites = time.time()
    for host, site_plots in plots_dict.items():
Julius Metz's avatar
Julius Metz committed
791 792
        with Path(plots_dir, host+'-plots.html').open(mode='w') as plots_file:
            plots_file.write(build_html(site_plots))
Julius Metz's avatar
Julius Metz committed
793
    print("write/build of websites took {:.1f}s".format(time.time() - time_sites))
Julius Metz's avatar
Julius Metz committed
794 795 796 797


if __name__ == '__main__':
    main()