Collectl2plotly.py 31.3 KB
Newer Older
1
import re
Julius Metz's avatar
Julius Metz committed
2
import gzip
Julius Metz's avatar
Julius Metz committed
3 4
import subprocess
from pathlib import Path
5 6
import copy
import time
7
import datetime
Julius Metz's avatar
Julius Metz committed
8 9
import multiprocessing as mp
import shutil
Julius Metz's avatar
Julius Metz committed
10
import importlib
Julius Metz's avatar
Julius Metz committed
11 12
import pickle
import json
Julius Metz's avatar
Julius Metz committed
13 14

import click
15 16
from yattag import Doc

Julius Metz's avatar
Julius Metz committed
17

18 19 20
import Collectl2plotly.filter_func as filter_func
import Collectl2plotly.value_merger as value_merger
import Collectl2plotly.Collectl2plotly_Plots_Config as default_plot_conf
Julius Metz's avatar
Julius Metz committed
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36


### Filter Configs

# registered filter functions
# filter functions must be in filter_func.py
# key  : Display name
# value: function name
FILTER_FUNCTIONS = {'hardvalue': 'filter_hardvalue', 'average': 'filter_average'}

# Default filter function if no explicit is given
# must be a display name of a function
DEFAULT_FILTER = 'average'

### validation Config

Julius Metz's avatar
Julius Metz committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
# Merger that exist and posible kwargs/parameter
MERGER = {
    'x2oneaddition_int': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
    },
    'x2oneaddition_float': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
    }
}

# validation infos of config
# key = config variablename to validate
# value = validator dict or type/tuple of types
# hints!!!!
# '__merger__' can be used as type for the merger tuple.
# in fixed_keys and 'changing_keys' the 'validator' need not be given if sub-elements are not to be validated.
#
# validator dict has follow structur:
# 'type' : required, is the possible type/s of the obj that are valid
# 'emptyable': list/dict/tuple can be empty if emptyable is True. Default: False
# 'subvalue_validator': is a validatordict or type/tuple of types to check all elems are valid if type is list/tuple
# 'fixed_keys': only if type is dict. takes a dict:
#               key: a known key of the current dict that is checked
#               value: dict with following possible keys:
#                     'required': if False the key must not in the dict Default: True
#                     'validator':  is a validatordict or type/tuple of types to validate the value
# 'changing_keys': only if type is dict. takes a dict:
#               'key_type': type of the keys in the dict current dict
#               'validator':  is a validatordict or type/tuple of types to validate the values of the dict
#
VALIDATIONS_INFOS = {
    'NEEDED_VALUES_DEFAULTS': {
        'type': dict,
        'fixed_keys': {
            'default_base_value': {},
            'default_merger': {'validator': '__merger__'},
        },
    },
    'NEEDED_VALUES': {
        'type': dict,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': dict,
                'fixed_keys': {
                    'keys': {'validator': {'type': list, 'subvalue_validator': str}},
                    'merger': {'validator': '__merger__'},
                    'base_value': {'required': False}
                },
            },
        },
    },
    'PLOTS_CONFIG': {
        'type': list,
        'subvalue_validator': {
            'type': dict,
            'fixed_keys': {
                'name': {'validator': str},
                'needed_key': {'validator': str},
                'relative-absolute_xaxis': {'required': False, 'validator': bool},
                'plotly_settings': {'validator': dict},
            },
        },
    },
    'NAME_SPEZIAL_PARAMETER': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': list,
                'emptyable':True,
                'subvalue_validator': str,
            },
        },
    },
    'COMAND_BLACKLIST': {
        'type': list,
        'emptyable': True,
        'subvalue_validator': str,
    },
    'PLOTLY_COLORS': {
        'type': list,
        'subvalue_validator': str,
    },
    'PLOTLY_STATIC_COLORS': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': str,
        },
    },
}

Julius Metz's avatar
WC  
Julius Metz committed
136

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
### Other

# config for find best axis interval and unit
# from smallest unit to greatest
# intervals = possible intervals of the axis(needed)
# max = conversion value if not given greatest supported unit
# unit = str with unit name
AXIS_INTERVAL_REL = [
    {
        'max': 60,
        'intervals': [1, 5, 10],
        'unit': 'sec',
    },
    {
        'max': 60,
        'intervals': [0.5, 1, 5, 10],
        'unit': 'min',
    },
    {
        'intervals': [0.25 ,0.5, 1, 5, 10],
        'unit': 'h',
    }
]

Julius Metz's avatar
Julius Metz committed
161 162 163
#pattern for split of commands (with \" and \ )
FIELDS_PATTERN = re.compile(r'(?:(?:\s*[^\\]|\A)\"(.*?[^\\]|)\"|(?:\s+|\A)(?=[^\s])(.*?[^\\])(?= |\Z))')
#(?:\"(.*?)\"|(\S+))
164 165


Julius Metz's avatar
Julius Metz committed
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271

def is_merger(to_check):
    """Check is given obj a valid merger. Possible merger are given in MERGER.

    Arguments:
        to_check {unkown} -- obj to test

    Returns:
        (bool, str) -- return true if it is a merger false and error if not
    """
    try:
        name, mergerkwargs = to_check
    except ValueError:
        return False, "'{0}' has no right merger structur expected: (name[str], kwargs[dict])".format(to_check)

    if isinstance(name, str) and isinstance(mergerkwargs, dict):
        if name in MERGER:
            for kwarg_name, value in mergerkwargs.items():
                if kwarg_name in MERGER[name]:
                    if 'choices' in MERGER[name][kwarg_name]:
                        if not value in MERGER[name][kwarg_name]['choices']:
                            return False, '{0} {1} expect one of {2} got: {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['choices'], value
                            )
                    else:
                        if not isinstance(value, MERGER[name][kwarg_name]['type']):
                            return False, '{1} of {0} expect {2} not {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['type'], type(value)
                            )
                else:
                    return False, '{1} is unknown parameter of {0}'.format(name, kwarg_name)
        else:
            return False, '"{0}" is unknown merger'.format(name)
    else:
        return False, '"{0}" has no right merger structur expected: (name[str], kwargs[dict])'.format(to_check)

    return True, None


def check_valid(to_check, validationinfos):
    """check object is valid from the perspective of validationinfos

    Arguments:
        to_check {unkown} -- object to check is valid
        validationinfos {dict or type or tuple with types} -- infos what is  needed that to_check is valid

    Returns:
        (bool, str) -- return true if it is valid, false and error if not
    """
    if isinstance(validationinfos, dict):
        if validationinfos['type'] == '__merger__':
            return is_merger(to_check)
        else:
            if isinstance(to_check, validationinfos['type']):
                if validationinfos['type'] == dict:
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

                    for fixed_name, infos in validationinfos.get('fixed_keys', {}).items():
                        if fixed_name in to_check:
                            if 'validator' in infos:
                                valid, error_msg = check_valid(to_check[fixed_name], infos['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(fixed_name, error_msg)

                        elif infos.get('required', True):
                            return False, '{0} missing key: "{1}"'.format(to_check, fixed_name)

                    if 'changing_keys' in validationinfos:
                        for key, subvalue in to_check.items():
                            if not isinstance(key, validationinfos['changing_keys']['key_type']):
                                return False, '{0} key has wrong type expected {1} got {2}'.format(
                                    key,  validationinfos['changing_keys']['key_type'], type(key),
                                )
                            if 'validator' in validationinfos['changing_keys']:
                                valid, error_msg =  check_valid(subvalue, validationinfos['changing_keys']['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(key, error_msg)

                if validationinfos['type'] == list or validationinfos['type'] == tuple:
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

                    if 'subvalue_validator' in validationinfos:
                        for i, subvalue in enumerate(to_check):
                            valid, error_msg = check_valid(subvalue, validationinfos['subvalue_validator'])
                            if not valid:
                                return valid, '[{0}] => {1}'.format(i, error_msg)

            else:
                return False, '{0} is wrong type expected {1} got {2}'.format(
                        to_check, validationinfos['type'], type(to_check)
                    )

    else:
        if validationinfos == '__merger__':
            return is_merger(to_check)
        else:
            if not isinstance(to_check, validationinfos):
                return False, '{0} is {1} expected: {2}'.format(
                    to_check, type(to_check), validationinfos,
                )

    return True, None


272
def datestr2date(datestr):
Julius Metz's avatar
Julius Metz committed
273 274 275 276 277 278 279 280
    """Converts a "datestring" to a date Object.

    Arguments:
        datestr {str} -- string of a Date example: 20191224

    Returns:
        datetime.date -- date of the string
    """
281
    return datetime.date(
Julius Metz's avatar
Julius Metz committed
282 283 284 285
        int(datestr[:4]),
        int(datestr[4:6]),
        int(datestr[6:8]),
    )
286 287


Julius Metz's avatar
Julius Metz committed
288
def get_cmdname(cmd, spezial_parameters_of_all, coarsest=False):
289 290 291 292 293 294 295 296 297 298 299
    """search in complete commandstring the name of the skript or the command that is used

    Arguments:
        cmd {str} -- complete commandstring

    Keyword Arguments:
        coarsest {bool} -- return only the call function(example: bash, python) if True (default: {False})

    Returns:
        str -- new cmd name
    """
Matthias Lieber's avatar
Matthias Lieber committed
300
    # search for (for example) bash_function="python" in cmd="/usr/bin/python3 my_script.py"
Julius Metz's avatar
Julius Metz committed
301 302 303
    # split command with regex
    cmd_splited = [x[0] or x[1] for x in FIELDS_PATTERN.findall(cmd)]
    #shlex.split(cmd)
304 305
    bash_function = cmd_splited[0].split('/')[-1]
    bash_function = re.search(r'[^\W\n]+', bash_function).group(0)
Matthias Lieber's avatar
Matthias Lieber committed
306
    # check if bash_function is known, if not, return bash_function
Julius Metz's avatar
Julius Metz committed
307 308
    spezial_parameters = spezial_parameters_of_all.get(bash_function, None)
    if coarsest or spezial_parameters is None:
309 310
        return bash_function
    skip = False
Matthias Lieber's avatar
Matthias Lieber committed
311
    # search script/program name within the parameters and only return this without path
312 313 314 315
    for position, parameter in enumerate(cmd_splited[1:]):
        if skip:
            skip = False
            continue
Julius Metz's avatar
Julius Metz committed
316
        if parameter in spezial_parameters:
317 318 319 320
            skip = True
            continue
        if bash_function == 'bash' or bash_function == 'sh' and parameter == '-c':
            return bash_function + ' -c'
Julius Metz's avatar
Julius Metz committed
321
            # return cmd_splited[position+1]
322 323 324 325 326 327
        if parameter.startswith('-'):
            continue
        return parameter.split('/')[-1]
    return cmd


Julius Metz's avatar
Julius Metz committed
328
def parse_file(path, collectl, shorten_cmds, coarsest, config):
Julius Metz's avatar
Julius Metz committed
329 330 331 332 333 334 335 336
    """start subproccess collectl than parse the output and merge.
       After that build usefull dict from parsed data

    Arguments:
        path {Path} -- path to collectl file to parse
        collectl {str} -- collectl command
        shorten_cmds {bool} -- if True cmd will be shorted by get_cmdname
        coarsest {bool} -- parameter of get_cmdname
Julius Metz's avatar
Julius Metz committed
337 338
        config {dict} -- plots and merge Config

Julius Metz's avatar
Julius Metz committed
339 340

    Returns:
Julius Metz's avatar
Julius Metz committed
341 342 343
        (dict, dict) --  1. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                         2. data for filter function :
                         {'number_of_values': X, 'commands':{ cmd:{'number_of_values': X, metrics:SUM, ...}}, metrics:SUM, ...}
Julius Metz's avatar
Julius Metz committed
344
    """
345
    collectl_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
346
    # run collectl in playback mode and read output into list
347
    process = subprocess.run(
Matthias Lieber's avatar
Matthias Lieber committed
348
        [collectl, '-P', '-p', path, '-sZ'], stdout=subprocess.PIPE, check=True,
349
    )
Julius Metz's avatar
Julius Metz committed
350 351
    print('collectl make table took {:.1f}s'.format(
        time.time() - collectl_starttime))
352
    parsing_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
353
    # output contains all data!
Julius Metz's avatar
Julius Metz committed
354
    output = process.stdout.decode().splitlines()
Matthias Lieber's avatar
Matthias Lieber committed
355
    # get table head
Julius Metz's avatar
Julius Metz committed
356
    head = output.pop(0).split(' ')
357 358 359 360 361 362 363
    for possible_head in output[:]:
        if possible_head.startswith('#'):
            head = possible_head.split(' ')
            output.remove(possible_head)
        else:
            break

Matthias Lieber's avatar
Matthias Lieber committed
364
    # get template of an entry from the head
Julius Metz's avatar
Julius Metz committed
365
    head[0] = head[0][1:]
Julius Metz's avatar
Julius Metz committed
366 367
    head_indexes_dict = {
        head_title: index for index, head_title in enumerate(head)}
368 369
    empty_dict_with_value_titles = {
        value_title: copy.deepcopy(
Julius Metz's avatar
Julius Metz committed
370
            value_title_settings.get(
Julius Metz's avatar
Julius Metz committed
371
                'base_value', config['NEEDED_VALUES_DEFAULTS']['default_base_value']
Julius Metz's avatar
Julius Metz committed
372
            )
Julius Metz's avatar
Julius Metz committed
373
        ) for value_title, value_title_settings in config['NEEDED_VALUES'].items()
374
    }
Julius Metz's avatar
Julius Metz committed
375

Matthias Lieber's avatar
Matthias Lieber committed
376 377
    # parse all output lines
    entrys_data = {}
Julius Metz's avatar
WC  
Julius Metz committed
378
    cmd_cmdshort_dict = {}
Julius Metz's avatar
Julius Metz committed
379 380
    merger_lookup_dict = {}

Julius Metz's avatar
Julius Metz committed
381
    for entry in output:
Matthias Lieber's avatar
Matthias Lieber committed
382
        # split by ' ' (exclude command from splitting)
383
        splited_entry = entry.split(' ', len(head_indexes_dict)-1)
Matthias Lieber's avatar
Matthias Lieber committed
384
        # get command string and shorten
Julius Metz's avatar
Julius Metz committed
385
        cmd = splited_entry[-1]
Julius Metz's avatar
Julius Metz committed
386
        if shorten_cmds or coarsest:
Julius Metz's avatar
WC  
Julius Metz committed
387 388 389
            if cmd in cmd_cmdshort_dict:
                cmd = cmd_cmdshort_dict[cmd]
            else:
Julius Metz's avatar
Julius Metz committed
390
                short_cmd = get_cmdname(cmd, config['NAME_SPEZIAL_PARAMETER'], coarsest=coarsest)
Julius Metz's avatar
WC  
Julius Metz committed
391 392
                cmd_cmdshort_dict[cmd]= short_cmd
                cmd = short_cmd
Julius Metz's avatar
Julius Metz committed
393
        if cmd in config['COMAND_BLACKLIST']:
Julius Metz's avatar
Julius Metz committed
394
            continue
Matthias Lieber's avatar
Matthias Lieber committed
395
        # create dict for each command
Julius Metz's avatar
Julius Metz committed
396 397
        if not cmd in entrys_data:
            entrys_data[cmd] = {}
Matthias Lieber's avatar
Matthias Lieber committed
398
        # get datetime obj for current entry
Julius Metz's avatar
Julius Metz committed
399 400
        tmp_datetime = datetime.datetime.combine(
            datestr2date(splited_entry[head_indexes_dict['Date']]),
Matthias Lieber's avatar
Matthias Lieber committed
401
            datetime.time(*[int(n) for n in splited_entry[head_indexes_dict['Time']].split(':')]),
Julius Metz's avatar
Julius Metz committed
402
        )
Matthias Lieber's avatar
Matthias Lieber committed
403
        # if datetime not yet existing, add new entry from template
Julius Metz's avatar
Julius Metz committed
404
        if not tmp_datetime in entrys_data[cmd]:
Julius Metz's avatar
Julius Metz committed
405 406
            entrys_data[cmd][tmp_datetime] = pickle.loads(pickle.dumps(
                empty_dict_with_value_titles, -1))
Matthias Lieber's avatar
Matthias Lieber committed
407 408
        # get values from data as given in NEEDED_VALUES and run specified merger function
        # to merge multiple values with same timestamp or rescale (e.g. to GB)
Julius Metz's avatar
Julius Metz committed
409 410 411 412 413
        for value_title, value_title_settings in config['NEEDED_VALUES'].items():
            merger, merger_kwargs = value_title_settings.get('merger', config['NEEDED_VALUES_DEFAULTS']['default_merger'])
            if not merger in merger_lookup_dict:
                merger_lookup_dict[merger] = getattr(value_merger, merger)
            entrys_data[cmd][tmp_datetime][value_title] = merger_lookup_dict[merger](
Julius Metz's avatar
Julius Metz committed
414 415
                entrys_data[cmd][tmp_datetime][value_title],
                *[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
Julius Metz's avatar
Julius Metz committed
416
                **merger_kwargs
417
            )
Julius Metz's avatar
Julius Metz committed
418
    print('parsing/merge took {:.1f}s'.format(time.time() - parsing_starttime))
419
    dictbuild_starttime = time.time()
420

Matthias Lieber's avatar
Matthias Lieber committed
421 422
    # create lists entry_data_plotfriendly[cmd][metric] = [ values, ... ] with the actual data to plot
    # and sum up all metrics for each command to enable filtering of non-interesting commands later on
423
    entry_data_plotfriendly = {}
Julius Metz's avatar
Julius Metz committed
424
    plot_filter_data = pickle.loads(pickle.dumps(empty_dict_with_value_titles, -1))
425 426 427
    plot_filter_data['number_of_values'] = 0
    plot_filter_data['commands'] = {}
    for cmd, cmd_data in entrys_data.items():
Julius Metz's avatar
Julius Metz committed
428 429
        plot_filter_data['commands'][cmd] = pickle.loads(pickle.dumps(
            empty_dict_with_value_titles, -1))
430
        plot_filter_data['commands'][cmd]['number_of_values'] = 0
Julius Metz's avatar
Julius Metz committed
431
        entry_data_plotfriendly[cmd] = {key: [] for key in config['NEEDED_VALUES']}
432
        entry_data_plotfriendly[cmd]['datetime'] = []
Julius Metz's avatar
Julius Metz committed
433
        for cmd_data_time, cmd_data_values in cmd_data.items():
434 435 436 437 438 439 440 441
            entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
            for cmd_data_key, cmd_data_value in cmd_data_values.items():
                entry_data_plotfriendly[cmd][cmd_data_key].append(cmd_data_value)

                plot_filter_data['commands'][cmd][cmd_data_key] += cmd_data_value
                plot_filter_data['commands'][cmd]['number_of_values'] += 1
                plot_filter_data[cmd_data_key] += cmd_data_value
                plot_filter_data['number_of_values'] += 1
Julius Metz's avatar
Julius Metz committed
442 443 444 445
    print(
        'data dict/ filter_data_dict build took {:.1f}s'.format(
            time.time() - dictbuild_starttime),
    )
446
    return entry_data_plotfriendly, plot_filter_data
Julius Metz's avatar
Julius Metz committed
447

Julius Metz's avatar
Julius Metz committed
448

449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
def make_relative_xaxi(all_values):
    """build lists with relative values for xaxis

    Arguments:
        all_values {[datetime, ..]} -- list of all datetime objects for plot

    Returns:
        (list, list) -- 1.: list with the relative values that are displayed on the xaxis
                        2.: list with the datetime as str of the absolute values of xaxis
    """
    min_value = min(all_values)
    max_value = max(all_values)
    guideline_tickcounts = 10
    end_value = (max_value - min_value).total_seconds()
    conversion_factor = 1

    for current_unit in AXIS_INTERVAL_REL:
        if 'max' in current_unit and end_value > current_unit['max']:
            end_value /= current_unit['max']
            conversion_factor *= current_unit['max']
            continue

        distances = [
            max(end_value // interval - guideline_tickcounts, guideline_tickcounts - end_value // interval) \
                for interval in current_unit['intervals']
        ]
        interval = current_unit['intervals'][distances.index(min(distances))]
        xaxis_ticks = [interval * i for i in range(int(end_value/interval)+1)]
        if len(xaxis_ticks) == 1:
            xaxis_ticks.append(1)
        xaxis_ticks_values = [str(min_value + datetime.timedelta(seconds=tick * conversion_factor)) for tick in xaxis_ticks]
        return xaxis_ticks, xaxis_ticks_values, current_unit['unit']


def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key, relative_xaxis):
Julius Metz's avatar
WC  
Julius Metz committed
484
    """Build a list of dicts in Plotlyconf style for the diffrent traces with the data from cmds_data of needed_key
485
        than make plot dict in Plotly style and change '{host}' in titels.
Julius Metz's avatar
Julius Metz committed
486 487 488 489 490

    Arguments:
        cmds_data {dict} -- is the data for the plot example: {'Command': {'datetime':[...], 'cpu': [...]}}
        filter_info {dict} -- has the information which cmds will be shown example: {'cpu': [cmds]}
        cmd_color {dict} -- assign a fixed plot color to each cmd example:  {'Command': 'rgb(0, 0, 128)'}
Julius Metz's avatar
WC  
Julius Metz committed
491 492
        plot_settings {dict} -- settings for Plotly
        plotly_format_vars {dict} -- values for Plotly settings
Julius Metz's avatar
Julius Metz committed
493
        needed_key {str} -- key of cmds_data for the values to be use
494
        relative_xaxis {bool} -- if true add buttons to change xaxis to relative
Julius Metz's avatar
Julius Metz committed
495 496

    Returns:
497
        dict -- with plotly jsons
Julius Metz's avatar
Julius Metz committed
498 499 500
    """
    plot_data = []
    for cmd in filter_info[needed_key]:
Julius Metz's avatar
Julius Metz committed
501 502 503
        plot_data.append({
            'x': [str(date) for date in cmds_data[cmd]['datetime']],
            'y': cmds_data[cmd][needed_key],
Julius Metz's avatar
Julius Metz committed
504
            'name': cmd,
Julius Metz's avatar
Julius Metz committed
505
            'marker': {
Julius Metz's avatar
Julius Metz committed
506 507
                'color': cmd_color[cmd],
            },
Julius Metz's avatar
Julius Metz committed
508
            **plot_settings['data'],
Julius Metz's avatar
WC  
Julius Metz committed
509 510 511 512
        }
        )


513 514 515
    layout = pickle.loads(pickle.dumps(plot_settings.get('layout', {}), -1))

    # replace {host} in titles
Julius Metz's avatar
Julius Metz committed
516 517 518
    if 'title' in layout:
        if type(layout['title']) == str:
            layout['title'] = layout['title'].format(
Julius Metz's avatar
Julius Metz committed
519 520 521
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
522
            layout['title']['text'] = layout['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
523 524
                **plotly_format_vars,
            )
Julius Metz's avatar
WC  
Julius Metz committed
525

Julius Metz's avatar
Julius Metz committed
526 527 528
    if 'yaxis' in layout and 'title' in layout['yaxis']:
        if type(layout['yaxis']['title']) == str:
            layout['yaxis']['title'] = layout['yaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
529 530 531
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
532
            layout['yaxis']['title']['text'] = layout['yaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
533 534
                **plotly_format_vars,
            )
535
    xtitle = ''
Julius Metz's avatar
Julius Metz committed
536 537 538
    if 'xaxis' in layout and 'title' in layout['xaxis']:
        if type(layout['xaxis']['title']) == str:
            layout['xaxis']['title'] = layout['xaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
539 540
                **plotly_format_vars,
            )
541
            xtitle = layout['xaxis']['title']
Julius Metz's avatar
Julius Metz committed
542
        else:
Julius Metz's avatar
Julius Metz committed
543
            layout['xaxis']['title']['text'] = layout['xaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
544 545
                **plotly_format_vars,
            )
546
            xtitle = layout['xaxis']['title']['text']
Julius Metz's avatar
WC  
Julius Metz committed
547

548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
    # add buttons for switching xaxis if 'relative-absolute_xaxis' in PLOT_CONF is set.
    if relative_xaxis:
            ticks_name, ticks_value, unit = make_relative_xaxi(
                [date for cmd in filter_info[needed_key] for date in cmds_data[cmd]['datetime']],
            )
            if not 'xaxis' in layout:
                layout['xaxis'] = {}
            if not 'updatemenus' in layout:
                layout['updatemenus'] = []
            layout['updatemenus'].append({
                'type': 'buttons',
                'x': 0.6,
                'y': 1.15,
                'direction': 'left',
                'buttons': [
                    {'args':[{'xaxis':{'title': xtitle, 'tickvals': None}}],
                    'label':'absolute xaxis',
                    'method':'relayout'},
                    {'args':[{'xaxis': {'title': 'runtime in {}'.format(unit), 'tickvals': ticks_value, 'ticktext': ticks_name}}],
                    'label':'relative xaxis',
                    'method':'relayout'},
                ],
                'showactive':True,
            })

    return {'data': json.dumps(plot_data), 'layout': json.dumps(layout), 'config': json.dumps(plot_settings.get('config', {}))}
Julius Metz's avatar
Julius Metz committed
574 575 576


def build_html(plots_dict):
Julius Metz's avatar
Julius Metz committed
577 578 579 580 581 582 583 584
    """build html site with the plots in plotsdict

    Arguments:
        plots_dict {dict} -- key: name of plot value: [plot, ...]

    Returns:
        str -- html website
    """
585 586 587 588
    doc, tag, text = Doc().tagtext()
    doc.asis('<!DOCTYPE html>')
    with tag('html'):
        with tag('head'):
Julius Metz's avatar
Julius Metz committed
589 590
            with tag('script', src='plotly.js'):
                pass
Julius Metz's avatar
Julius Metz committed
591 592 593 594
            with tag('script'):
                doc.asis("document.onreadystatechange = () => {if (document.readyState === 'complete') {")
                for name, plots in plots_dict.items():
                    for i, plot in enumerate(plots):
595 596
                        doc.asis("Plotly.newPlot(document.getElementById('{name}-plot-{number}'), JSON.parse('{data}'), JSON.parse('{layout}'), JSON.parse('{config}'));"\
                            .format(name=name, number=i, data=plot['data'], layout=plot['layout'], config=plot['config'])
Julius Metz's avatar
Julius Metz committed
597 598
                        )
                doc.asis('}}')
599 600 601 602 603
        with tag('body'):
            with tag('div', id='index'):
                with tag('h2'):
                    text('Index:')
                with tag('ul'):
Julius Metz's avatar
Julius Metz committed
604
                    for name in plots_dict:
605 606 607 608 609
                        with tag('li'):
                            with tag('a', href='#'+name):
                                text(name)
            for name, plots in plots_dict.items():
                with tag('div', id=name):
Julius Metz's avatar
Julius Metz committed
610 611 612
                    for i in range(len(plots)):
                        with tag('div', id='{}-plot-{}'.format(name, i)):
                            pass
613 614 615 616

    return doc.getvalue()


Julius Metz's avatar
Julius Metz committed
617 618 619 620 621 622 623 624 625 626 627 628 629
def get_sources(sources):
    """collect all sources (.raw.gz files)

    Arguments:
        sources {tuple} -- tuple of files/directorys

    Returns:
        list -- list of all .raw.gz files which was found
    """
    source_paths = []
    for source in sources:
        source_path =  Path(source)
        if source_path.is_dir():
Julius Metz's avatar
Julius Metz committed
630
            source_paths.extend(source_path.glob('*.raw.gz'))
Julius Metz's avatar
Julius Metz committed
631 632 633 634 635
        elif source_path.is_file() and source.endswith('.raw.gz'):
            source_paths.append(source_path)
    return source_paths


Julius Metz's avatar
Julius Metz committed
636
def data_from_file(arguments):
Julius Metz's avatar
Julius Metz committed
637 638 639 640 641 642 643 644 645 646
    """make data and filter infos of one collectl file

    Arguments:
        arguments {tuple} -- is a tuple of arguments for the function:
                    path {Path} -- collectldata file path
                    collectl {str} -- collectl command
                    shorten_cmds {bool} -- para for parse_file
                    coarsest {bool} -- para for parse_file
                    filtercmds {bool} -- if True cmds will be filter else not
                    filtervalue {int} -- para for filter
Matthias Lieber's avatar
Matthias Lieber committed
647
                    filtertype {str} -- which filter will be called
Julius Metz's avatar
Julius Metz committed
648
                    config {dict} -- plots and merge Config
Julius Metz's avatar
Julius Metz committed
649
    Returns:
Julius Metz's avatar
Julius Metz committed
650 651 652
        (str, dict, dict) -- 1. hostname
                             2. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                             3. filter_infos = {metrics: [cmds, ...]
Julius Metz's avatar
Julius Metz committed
653
    """
Julius Metz's avatar
Julius Metz committed
654
    path, collectl, shorten_cmds, coarsest, filtercmds, filtervalue, filtertype, config = arguments
Julius Metz's avatar
Julius Metz committed
655 656 657 658 659
    host = ''
    with gzip.open(path, 'r') as f:
        for line in f:
            if line.startswith(b'# Host:'):
                host = re.search(r'# Host: *([^ ]+)', line.decode()).group(1)
Julius Metz's avatar
Julius Metz committed
660
    data, filter_data = parse_file(path, collectl, shorten_cmds, coarsest, config)
Julius Metz's avatar
Julius Metz committed
661 662 663
    if filtercmds:
        filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
    else:
Julius Metz's avatar
Julius Metz committed
664
        filter_infos = {key: list(data.keys()) for key in config['NEEDED_VALUES']}
Julius Metz's avatar
Julius Metz committed
665 666
    return host, data, filter_infos

667

Julius Metz's avatar
Julius Metz committed
668 669 670
@click.command(help='Generate htmlfiles with Plotlyplots with data from collectlfiles(".raw.gz")')
@click.option('--source', '-s', multiple=True, default=['.'], show_default=True, type=click.Path(exists=True), help='source for the plots. (.raw.gz file or directory with .raw.gz) multiple useable')
@click.option('--collectl', '-c', default='collectl', show_default=True, help='collectl command')
Julius Metz's avatar
Julius Metz committed
671
@click.option('--plotlypath', '-p', type=click.Path(exists=True), help='path to plotly.js')
Julius Metz's avatar
Julius Metz committed
672
@click.option('--destination', '-d', default='.', type=click.Path(exists=True), show_default=True, help='path to directory where directory with plots will be created')
Julius Metz's avatar
Julius Metz committed
673
@click.option('--configpath', default=None, type=click.Path(exists=True), help='python file with plot and merge infos see doku for detail')
Julius Metz's avatar
Julius Metz committed
674
@click.option('--shorten/--notshorten', default=True, help='commands will be shorted only to name')
675
@click.option('--coarsest', is_flag=True, help='commands will be shorted only to type (bash, perl, ...)')
Julius Metz's avatar
Julius Metz committed
676
@click.option('--filtercmds/--notfiltercmds', default=True, help='filtering or not')
Matthias Lieber's avatar
Matthias Lieber committed
677
@click.option('--filtervalue', help='Parameter which is given to the filter.')
Julius Metz's avatar
Julius Metz committed
678 679 680 681 682
@click.option('--filtertype',
              type=click.Choice(FILTER_FUNCTIONS.keys(), case_sensitive=False),
              default=DEFAULT_FILTER, show_default=True,
              help='Filter which is to be used.',
              )
Julius Metz's avatar
Julius Metz committed
683 684 685
@click.option('--force', is_flag=True, help='override existing plot directory if exist')
def main(source, collectl, plotlypath, destination, configpath,
         shorten, coarsest, filtercmds, filtervalue, filtertype, force):
Julius Metz's avatar
Julius Metz committed
686
    source_paths = get_sources(source)
687

Julius Metz's avatar
Julius Metz committed
688
    if not source_paths:
Julius Metz's avatar
Julius Metz committed
689 690 691 692 693 694 695 696 697
        print('no valid source found')
        exit(1)

    if not Path(destination).is_dir():
        print('destination is no valid directory')
        exit(1)

    plots_dir = Path(destination, 'collectlplots')
    if plots_dir.is_dir():
Julius Metz's avatar
Julius Metz committed
698 699 700 701 702 703 704 705 706
        if force:
            shutil.rmtree(plots_dir)
        else:
            print('in destination "collectlplots" already exist')
            exit(1)

    config_module = default_plot_conf
    if configpath:
        if configpath.endswith('.py'):
707
            config_module = importlib.machinery.SourceFileLoader('config', configpath).load_module()
Julius Metz's avatar
Julius Metz committed
708 709 710 711 712 713
        else:
            print('given config isn`t a ".py" Python file')
            exit(1)

    # validate config and make dict from values
    config = {}
Julius Metz's avatar
Julius Metz committed
714 715
    a = time.time()
    for config_var_name, validator_dict in VALIDATIONS_INFOS.items():
Julius Metz's avatar
Julius Metz committed
716 717 718 719 720
        try:
            config[config_var_name] = getattr(config_module, config_var_name)
        except AttributeError:
            print('{} missing in config'.format(config_var_name))
            exit(1)
Julius Metz's avatar
Julius Metz committed
721 722 723 724
        valid, errormsg = check_valid(config[config_var_name], validator_dict)
        if not valid:
            print('In {0}: \n{1}'.format(config_var_name, errormsg))
            exit(1)
725

Julius Metz's avatar
Julius Metz committed
726 727 728
    data_colllect_functions = []
    for source_path in source_paths:
        data_colllect_functions.append((source_path,
Julius Metz's avatar
Julius Metz committed
729 730 731 732 733 734
                                        collectl,
                                        shorten,
                                        coarsest,
                                        filtercmds,
                                        filtervalue,
                                        FILTER_FUNCTIONS[filtertype],
Julius Metz's avatar
Julius Metz committed
735
                                        config,
Julius Metz's avatar
Julius Metz committed
736
                                        ))
737

Matthias Lieber's avatar
Matthias Lieber committed
738
    # use multiprocessing to parse all collectl output files independently from each other in parallel
Julius Metz's avatar
Julius Metz committed
739 740 741 742
    pool = mp.Pool(min(len(data_colllect_functions), mp.cpu_count()))
    results = pool.map(data_from_file, data_colllect_functions)
    pool.close()
    hosts_data = {}
Julius Metz's avatar
Julius Metz committed
743
    cmd_all = set()
Julius Metz's avatar
Julius Metz committed
744

Julius Metz's avatar
Julius Metz committed
745 746
    for host, data, filter_infos in results:
        hosts_data[host] = {'data': data, 'filter_infos': filter_infos}
Julius Metz's avatar
Julius Metz committed
747
        cmd_all.update([cmd for cmds in filter_infos.values() for cmd in cmds])
Julius Metz's avatar
Julius Metz committed
748

Julius Metz's avatar
Julius Metz committed
749 750
    cmd_colors = {cmd: config['PLOTLY_COLORS'][i % len(config['PLOTLY_COLORS'])] for i, cmd in enumerate(sorted(list(cmd_all)))}
    cmd_colors.update(config['PLOTLY_STATIC_COLORS'])
Julius Metz's avatar
Julius Metz committed
751

Matthias Lieber's avatar
Matthias Lieber committed
752
    # for each host and each plot (as given in config) call make_plot to create html div with plotly
Julius Metz's avatar
Julius Metz committed
753 754 755
    start_plots_build = time.time()
    plots_dict = {}
    for host, host_data in hosts_data.items():
Julius Metz's avatar
Julius Metz committed
756
        plots_dict[host] = {plot_config['name']: [] for plot_config in config['PLOTS_CONFIG']}
Julius Metz's avatar
Julius Metz committed
757
        for plot_config in config['PLOTS_CONFIG']:
Matthias Lieber's avatar
Matthias Lieber committed
758
            plots_dict[host][plot_config['name']].append( make_plot(
Julius Metz's avatar
Julius Metz committed
759 760 761 762 763 764
                host_data['data'],
                host_data['filter_infos'],
                cmd_colors,
                plot_config['plotly_settings'],
                {'host': host},
                plot_config['needed_key'],
765
                plot_config.get('relative-absolute_xaxis', False),
Julius Metz's avatar
Julius Metz committed
766 767 768
            ))
    print("plots build in {:.1f}s".format(time.time() - start_plots_build))

Julius Metz's avatar
Julius Metz committed
769

Matthias Lieber's avatar
Matthias Lieber committed
770
    # create output directory and copy plotly.js
Julius Metz's avatar
Julius Metz committed
771
    plots_dir.mkdir()
Julius Metz's avatar
Julius Metz committed
772 773
    if plotlypath is None:
        plotlypath = Path(__file__).with_name('plotly-latest.min.js')
Julius Metz's avatar
Julius Metz committed
774
    try:
Julius Metz's avatar
Julius Metz committed
775
        shutil.copy(plotlypath, str(Path(plots_dir, 'plotly.js')))
Julius Metz's avatar
Julius Metz committed
776 777 778
    except shutil.SameFileError:
        pass

Matthias Lieber's avatar
Matthias Lieber committed
779
    # for each host create html file with plots
Julius Metz's avatar
Julius Metz committed
780 781
    time_sites = time.time()
    for host, site_plots in plots_dict.items():
Julius Metz's avatar
Julius Metz committed
782 783
        with Path(plots_dir, host+'-plots.html').open(mode='w') as plots_file:
            plots_file.write(build_html(site_plots))
Julius Metz's avatar
Julius Metz committed
784
    print("write/build of websites took {:.1f}s".format(time.time() - time_sites))
Julius Metz's avatar
Julius Metz committed
785 786 787 788


if __name__ == '__main__':
    main()