Collectl2plotly.py 33.3 KB
Newer Older
1
import re
Julius Metz's avatar
Julius Metz committed
2
import gzip
Julius Metz's avatar
Julius Metz committed
3 4
import subprocess
from pathlib import Path
5 6
import copy
import time
7
import datetime
Julius Metz's avatar
Julius Metz committed
8 9
import multiprocessing as mp
import shutil
Julius Metz's avatar
Julius Metz committed
10
import importlib
Julius Metz's avatar
Julius Metz committed
11 12
import pickle
import json
Julius Metz's avatar
Julius Metz committed
13 14

import click
15 16
from yattag import Doc

Julius Metz's avatar
Julius Metz committed
17

18 19 20
import Collectl2plotly.filter_func as filter_func
import Collectl2plotly.value_merger as value_merger
import Collectl2plotly.Collectl2plotly_Plots_Config as default_plot_conf
Julius Metz's avatar
Julius Metz committed
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36


### Filter Configs

# registered filter functions
# filter functions must be in filter_func.py
# key  : Display name
# value: function name
FILTER_FUNCTIONS = {'hardvalue': 'filter_hardvalue', 'average': 'filter_average'}

# Default filter function if no explicit is given
# must be a display name of a function
DEFAULT_FILTER = 'average'

### validation Config

37 38 39 40 41 42 43 44
# Merger that exist and possible kwargs/parameter
# key   = merger_func name
# value = dict with kwarguments of merger_func 
# kwargument dict:
# key = argument name
# value = 1. 'type' with a type obj or a tuple of type objs 
#         or 
#         2. 'choices' list of possible choices for the kwargument
Julius Metz's avatar
Julius Metz committed
45 46 47
MERGER = {
    'x2oneaddition_int': {
        'roundingpoint': {'type':int},
48
        'operator': {'choices': ['+', '-', '*', '/', '%']},
Julius Metz's avatar
Julius Metz committed
49 50 51 52 53 54
        'operator_value': {'type': (int, float)},
    },
    'x2oneaddition_float': {
        'roundingpoint': {'type':int},
        'operator': {'type': str, 'choices': ['+', '-', '*', '/', '%']},
        'operator_value': {'type': (int, float)},
55
    },
Julius Metz's avatar
Julius Metz committed
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
}

# validation infos of config
# key = config variablename to validate
# value = validator dict or type/tuple of types
# hints!!!!
# '__merger__' can be used as type for the merger tuple.
# in fixed_keys and 'changing_keys' the 'validator' need not be given if sub-elements are not to be validated.
#
# validator dict has follow structur:
# 'type' : required, is the possible type/s of the obj that are valid
# 'emptyable': list/dict/tuple can be empty if emptyable is True. Default: False
# 'subvalue_validator': is a validatordict or type/tuple of types to check all elems are valid if type is list/tuple
# 'fixed_keys': only if type is dict. takes a dict:
#               key: a known key of the current dict that is checked
#               value: dict with following possible keys:
#                     'required': if False the key must not in the dict Default: True
#                     'validator':  is a validatordict or type/tuple of types to validate the value
# 'changing_keys': only if type is dict. takes a dict:
#               'key_type': type of the keys in the dict current dict
#               'validator':  is a validatordict or type/tuple of types to validate the values of the dict
#
VALIDATIONS_INFOS = {
    'NEEDED_VALUES_DEFAULTS': {
        'type': dict,
        'fixed_keys': {
            'default_base_value': {},
            'default_merger': {'validator': '__merger__'},
        },
    },
    'NEEDED_VALUES': {
        'type': dict,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': dict,
                'fixed_keys': {
                    'keys': {'validator': {'type': list, 'subvalue_validator': str}},
                    'merger': {'validator': '__merger__'},
                    'base_value': {'required': False}
                },
            },
        },
    },
    'PLOTS_CONFIG': {
        'type': list,
        'subvalue_validator': {
            'type': dict,
            'fixed_keys': {
                'name': {'validator': str},
                'needed_key': {'validator': str},
                'relative-absolute_xaxis': {'required': False, 'validator': bool},
                'plotly_settings': {'validator': dict},
            },
        },
    },
    'NAME_SPEZIAL_PARAMETER': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': {
                'type': list,
                'emptyable':True,
                'subvalue_validator': str,
            },
        },
    },
    'COMAND_BLACKLIST': {
        'type': list,
        'emptyable': True,
        'subvalue_validator': str,
    },
    'PLOTLY_COLORS': {
        'type': list,
        'subvalue_validator': str,
    },
    'PLOTLY_STATIC_COLORS': {
        'type': dict,
        'emptyable': True,
        'changing_keys': {
            'key_type': str,
            'validator': str,
        },
    },
}

Julius Metz's avatar
WC  
Julius Metz committed
143

144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
### Other

# config for find best axis interval and unit
# from smallest unit to greatest
# intervals = possible intervals of the axis(needed)
# max = conversion value if not given greatest supported unit
# unit = str with unit name
AXIS_INTERVAL_REL = [
    {
        'max': 60,
        'intervals': [1, 5, 10],
        'unit': 'sec',
    },
    {
        'max': 60,
        'intervals': [0.5, 1, 5, 10],
        'unit': 'min',
    },
    {
        'intervals': [0.25 ,0.5, 1, 5, 10],
        'unit': 'h',
    }
]

Julius Metz's avatar
Julius Metz committed
168 169 170
#pattern for split of commands (with \" and \ )
FIELDS_PATTERN = re.compile(r'(?:(?:\s*[^\\]|\A)\"(.*?[^\\]|)\"|(?:\s+|\A)(?=[^\s])(.*?[^\\])(?= |\Z))')
#(?:\"(.*?)\"|(\S+))
171 172


Julius Metz's avatar
Julius Metz committed
173 174 175 176 177 178 179 180 181 182

def is_merger(to_check):
    """Check is given obj a valid merger. Possible merger are given in MERGER.

    Arguments:
        to_check {unkown} -- obj to test

    Returns:
        (bool, str) -- return true if it is a merger false and error if not
    """
183 184

    # check is tuple with 2 elems
Julius Metz's avatar
Julius Metz committed
185 186 187 188 189
    try:
        name, mergerkwargs = to_check
    except ValueError:
        return False, "'{0}' has no right merger structur expected: (name[str], kwargs[dict])".format(to_check)

190
    #check name and mergerkwargs has right types
Julius Metz's avatar
Julius Metz committed
191
    if isinstance(name, str) and isinstance(mergerkwargs, dict):
192
        # check is the given merger known
Julius Metz's avatar
Julius Metz committed
193
        if name in MERGER:
194
            # check every kwarg of the given merger is it known and is right value for it given.
Julius Metz's avatar
Julius Metz committed
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
            for kwarg_name, value in mergerkwargs.items():
                if kwarg_name in MERGER[name]:
                    if 'choices' in MERGER[name][kwarg_name]:
                        if not value in MERGER[name][kwarg_name]['choices']:
                            return False, '{0} {1} expect one of {2} got: {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['choices'], value
                            )
                    else:
                        if not isinstance(value, MERGER[name][kwarg_name]['type']):
                            return False, '{1} of {0} expect {2} not {3}'.format(
                                name, kwarg_name, MERGER[name][kwarg_name]['type'], type(value)
                            )
                else:
                    return False, '{1} is unknown parameter of {0}'.format(name, kwarg_name)
        else:
            return False, '"{0}" is unknown merger'.format(name)
    else:
        return False, '"{0}" has no right merger structur expected: (name[str], kwargs[dict])'.format(to_check)

    return True, None


def check_valid(to_check, validationinfos):
    """check object is valid from the perspective of validationinfos

    Arguments:
        to_check {unkown} -- object to check is valid
222
        validationinfos {validationdict(see comment of VALIDATIONS_INFOS) or type or tuple with types} -- infos what is needed that to_check is valid
Julius Metz's avatar
Julius Metz committed
223 224 225 226

    Returns:
        (bool, str) -- return true if it is valid, false and error if not
    """
227
    # check is validationinfos a validationdict or only a type/tuple of types
Julius Metz's avatar
Julius Metz committed
228 229 230 231
    if isinstance(validationinfos, dict):
        if validationinfos['type'] == '__merger__':
            return is_merger(to_check)
        else:
232
            # check is given elem expected type
Julius Metz's avatar
Julius Metz committed
233
            if isinstance(to_check, validationinfos['type']):
234
                # if elem dict check special cases of it
235
                if type(to_check) == dict:
236
                    # check is elem is empty and is this ok
Julius Metz's avatar
Julius Metz committed
237 238 239
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

240
                    # check special keys and there values if 'fixed_keys' is given
Julius Metz's avatar
Julius Metz committed
241 242 243 244 245 246 247 248 249 250
                    for fixed_name, infos in validationinfos.get('fixed_keys', {}).items():
                        if fixed_name in to_check:
                            if 'validator' in infos:
                                valid, error_msg = check_valid(to_check[fixed_name], infos['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(fixed_name, error_msg)

                        elif infos.get('required', True):
                            return False, '{0} missing key: "{1}"'.format(to_check, fixed_name)

251
                    # check all keys and there values if 'changing_keys' is given
Julius Metz's avatar
Julius Metz committed
252 253 254 255 256 257 258 259 260 261
                    if 'changing_keys' in validationinfos:
                        for key, subvalue in to_check.items():
                            if not isinstance(key, validationinfos['changing_keys']['key_type']):
                                return False, '{0} key has wrong type expected {1} got {2}'.format(
                                    key,  validationinfos['changing_keys']['key_type'], type(key),
                                )
                            if 'validator' in validationinfos['changing_keys']:
                                valid, error_msg =  check_valid(subvalue, validationinfos['changing_keys']['validator'])
                                if not valid:
                                    return valid, '[{0}] => {1}'.format(key, error_msg)
262 263 264
                # if elem list or tuple check special cases of it                 
                elif isinstance(to_check, (list, tuple)):
                    # check is elem is empty and is this ok
Julius Metz's avatar
Julius Metz committed
265 266 267
                    if not validationinfos.get('emptyable', False) and not len(to_check):
                        return False, '{0} can not be empty!'.format(to_check)

268
                    # validate all values of tuple/list if 'subvalue_validator' is given
Julius Metz's avatar
Julius Metz committed
269 270 271 272 273 274 275 276 277 278 279 280
                    if 'subvalue_validator' in validationinfos:
                        for i, subvalue in enumerate(to_check):
                            valid, error_msg = check_valid(subvalue, validationinfos['subvalue_validator'])
                            if not valid:
                                return valid, '[{0}] => {1}'.format(i, error_msg)

            else:
                return False, '{0} is wrong type expected {1} got {2}'.format(
                        to_check, validationinfos['type'], type(to_check)
                    )

    else:
281
        # check it is the special case that is the type is '__merger__'
Julius Metz's avatar
Julius Metz committed
282 283 284
        if validationinfos == '__merger__':
            return is_merger(to_check)
        else:
285
            # check is given elem expected type
Julius Metz's avatar
Julius Metz committed
286 287 288 289 290 291 292 293
            if not isinstance(to_check, validationinfos):
                return False, '{0} is {1} expected: {2}'.format(
                    to_check, type(to_check), validationinfos,
                )

    return True, None


294
def datestr2date(datestr):
Julius Metz's avatar
Julius Metz committed
295 296 297 298 299 300 301 302
    """Converts a "datestring" to a date Object.

    Arguments:
        datestr {str} -- string of a Date example: 20191224

    Returns:
        datetime.date -- date of the string
    """
303
    return datetime.date(
Julius Metz's avatar
Julius Metz committed
304 305 306 307
        int(datestr[:4]),
        int(datestr[4:6]),
        int(datestr[6:8]),
    )
308 309


Julius Metz's avatar
Julius Metz committed
310
def get_cmdname(cmd, spezial_parameters_of_all, coarsest=False):
311 312 313 314 315 316 317 318 319 320 321
    """search in complete commandstring the name of the skript or the command that is used

    Arguments:
        cmd {str} -- complete commandstring

    Keyword Arguments:
        coarsest {bool} -- return only the call function(example: bash, python) if True (default: {False})

    Returns:
        str -- new cmd name
    """
Matthias Lieber's avatar
Matthias Lieber committed
322
    # search for (for example) bash_function="python" in cmd="/usr/bin/python3 my_script.py"
Julius Metz's avatar
Julius Metz committed
323 324 325
    # split command with regex
    cmd_splited = [x[0] or x[1] for x in FIELDS_PATTERN.findall(cmd)]
    #shlex.split(cmd)
326 327
    bash_function = cmd_splited[0].split('/')[-1]
    bash_function = re.search(r'[^\W\n]+', bash_function).group(0)
Matthias Lieber's avatar
Matthias Lieber committed
328
    # check if bash_function is known, if not, return bash_function
Julius Metz's avatar
Julius Metz committed
329 330
    spezial_parameters = spezial_parameters_of_all.get(bash_function, None)
    if coarsest or spezial_parameters is None:
331 332
        return bash_function
    skip = False
Matthias Lieber's avatar
Matthias Lieber committed
333
    # search script/program name within the parameters and only return this without path
334 335 336 337
    for position, parameter in enumerate(cmd_splited[1:]):
        if skip:
            skip = False
            continue
Julius Metz's avatar
Julius Metz committed
338
        if parameter in spezial_parameters:
339 340 341 342
            skip = True
            continue
        if bash_function == 'bash' or bash_function == 'sh' and parameter == '-c':
            return bash_function + ' -c'
Julius Metz's avatar
Julius Metz committed
343
            # return cmd_splited[position+1]
344 345 346 347 348 349
        if parameter.startswith('-'):
            continue
        return parameter.split('/')[-1]
    return cmd


Julius Metz's avatar
Julius Metz committed
350
def parse_file(path, collectl, shorten_cmds, coarsest, config):
Julius Metz's avatar
Julius Metz committed
351 352 353 354 355 356 357 358
    """start subproccess collectl than parse the output and merge.
       After that build usefull dict from parsed data

    Arguments:
        path {Path} -- path to collectl file to parse
        collectl {str} -- collectl command
        shorten_cmds {bool} -- if True cmd will be shorted by get_cmdname
        coarsest {bool} -- parameter of get_cmdname
Julius Metz's avatar
Julius Metz committed
359 360
        config {dict} -- plots and merge Config

Julius Metz's avatar
Julius Metz committed
361 362

    Returns:
Julius Metz's avatar
Julius Metz committed
363 364 365
        (dict, dict) --  1. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                         2. data for filter function :
                         {'number_of_values': X, 'commands':{ cmd:{'number_of_values': X, metrics:SUM, ...}}, metrics:SUM, ...}
Julius Metz's avatar
Julius Metz committed
366
    """
367
    collectl_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
368
    # run collectl in playback mode and read output into list
369
    process = subprocess.run(
Matthias Lieber's avatar
Matthias Lieber committed
370
        [collectl, '-P', '-p', path, '-sZ'], stdout=subprocess.PIPE, check=True,
371
    )
Julius Metz's avatar
Julius Metz committed
372 373
    print('collectl make table took {:.1f}s'.format(
        time.time() - collectl_starttime))
374
    parsing_starttime = time.time()
Matthias Lieber's avatar
Matthias Lieber committed
375
    # output contains all data!
Julius Metz's avatar
Julius Metz committed
376
    output = process.stdout.decode().splitlines()
Matthias Lieber's avatar
Matthias Lieber committed
377
    # get table head
Julius Metz's avatar
Julius Metz committed
378
    head = output.pop(0).split(' ')
379 380 381 382 383 384 385
    for possible_head in output[:]:
        if possible_head.startswith('#'):
            head = possible_head.split(' ')
            output.remove(possible_head)
        else:
            break

Matthias Lieber's avatar
Matthias Lieber committed
386
    # get template of an entry from the head
Julius Metz's avatar
Julius Metz committed
387
    head[0] = head[0][1:]
Julius Metz's avatar
Julius Metz committed
388 389
    head_indexes_dict = {
        head_title: index for index, head_title in enumerate(head)}
390 391
    empty_dict_with_value_titles = {
        value_title: copy.deepcopy(
Julius Metz's avatar
Julius Metz committed
392
            value_title_settings.get(
Julius Metz's avatar
Julius Metz committed
393
                'base_value', config['NEEDED_VALUES_DEFAULTS']['default_base_value']
Julius Metz's avatar
Julius Metz committed
394
            )
Julius Metz's avatar
Julius Metz committed
395
        ) for value_title, value_title_settings in config['NEEDED_VALUES'].items()
396
    }
Julius Metz's avatar
Julius Metz committed
397

Matthias Lieber's avatar
Matthias Lieber committed
398 399
    # parse all output lines
    entrys_data = {}
Julius Metz's avatar
WC  
Julius Metz committed
400
    cmd_cmdshort_dict = {}
Julius Metz's avatar
Julius Metz committed
401 402
    merger_lookup_dict = {}

Julius Metz's avatar
Julius Metz committed
403
    for entry in output:
Matthias Lieber's avatar
Matthias Lieber committed
404
        # split by ' ' (exclude command from splitting)
405
        splited_entry = entry.split(' ', len(head_indexes_dict)-1)
Matthias Lieber's avatar
Matthias Lieber committed
406
        # get command string and shorten
Julius Metz's avatar
Julius Metz committed
407
        cmd = splited_entry[-1]
Julius Metz's avatar
Julius Metz committed
408
        if shorten_cmds or coarsest:
Julius Metz's avatar
WC  
Julius Metz committed
409 410 411
            if cmd in cmd_cmdshort_dict:
                cmd = cmd_cmdshort_dict[cmd]
            else:
Julius Metz's avatar
Julius Metz committed
412
                short_cmd = get_cmdname(cmd, config['NAME_SPEZIAL_PARAMETER'], coarsest=coarsest)
Julius Metz's avatar
WC  
Julius Metz committed
413 414
                cmd_cmdshort_dict[cmd]= short_cmd
                cmd = short_cmd
Julius Metz's avatar
Julius Metz committed
415
        if cmd in config['COMAND_BLACKLIST']:
Julius Metz's avatar
Julius Metz committed
416
            continue
Matthias Lieber's avatar
Matthias Lieber committed
417
        # create dict for each command
Julius Metz's avatar
Julius Metz committed
418 419
        if not cmd in entrys_data:
            entrys_data[cmd] = {}
Matthias Lieber's avatar
Matthias Lieber committed
420
        # get datetime obj for current entry
Julius Metz's avatar
Julius Metz committed
421 422
        tmp_datetime = datetime.datetime.combine(
            datestr2date(splited_entry[head_indexes_dict['Date']]),
Matthias Lieber's avatar
Matthias Lieber committed
423
            datetime.time(*[int(n) for n in splited_entry[head_indexes_dict['Time']].split(':')]),
Julius Metz's avatar
Julius Metz committed
424
        )
Matthias Lieber's avatar
Matthias Lieber committed
425
        # if datetime not yet existing, add new entry from template
Julius Metz's avatar
Julius Metz committed
426
        if not tmp_datetime in entrys_data[cmd]:
Julius Metz's avatar
Julius Metz committed
427 428
            entrys_data[cmd][tmp_datetime] = pickle.loads(pickle.dumps(
                empty_dict_with_value_titles, -1))
Matthias Lieber's avatar
Matthias Lieber committed
429 430
        # get values from data as given in NEEDED_VALUES and run specified merger function
        # to merge multiple values with same timestamp or rescale (e.g. to GB)
Julius Metz's avatar
Julius Metz committed
431 432 433 434 435
        for value_title, value_title_settings in config['NEEDED_VALUES'].items():
            merger, merger_kwargs = value_title_settings.get('merger', config['NEEDED_VALUES_DEFAULTS']['default_merger'])
            if not merger in merger_lookup_dict:
                merger_lookup_dict[merger] = getattr(value_merger, merger)
            entrys_data[cmd][tmp_datetime][value_title] = merger_lookup_dict[merger](
Julius Metz's avatar
Julius Metz committed
436 437
                entrys_data[cmd][tmp_datetime][value_title],
                *[splited_entry[head_indexes_dict[key]] for key in value_title_settings['keys']],
Julius Metz's avatar
Julius Metz committed
438
                **merger_kwargs
439
            )
Julius Metz's avatar
Julius Metz committed
440
    print('parsing/merge took {:.1f}s'.format(time.time() - parsing_starttime))
441
    dictbuild_starttime = time.time()
442

Matthias Lieber's avatar
Matthias Lieber committed
443 444
    # create lists entry_data_plotfriendly[cmd][metric] = [ values, ... ] with the actual data to plot
    # and sum up all metrics for each command to enable filtering of non-interesting commands later on
445
    entry_data_plotfriendly = {}
Julius Metz's avatar
Julius Metz committed
446
    plot_filter_data = pickle.loads(pickle.dumps(empty_dict_with_value_titles, -1))
447 448 449
    plot_filter_data['number_of_values'] = 0
    plot_filter_data['commands'] = {}
    for cmd, cmd_data in entrys_data.items():
Julius Metz's avatar
Julius Metz committed
450 451
        plot_filter_data['commands'][cmd] = pickle.loads(pickle.dumps(
            empty_dict_with_value_titles, -1))
452
        plot_filter_data['commands'][cmd]['number_of_values'] = 0
Julius Metz's avatar
Julius Metz committed
453
        entry_data_plotfriendly[cmd] = {key: [] for key in config['NEEDED_VALUES']}
454
        entry_data_plotfriendly[cmd]['datetime'] = []
Julius Metz's avatar
Julius Metz committed
455
        for cmd_data_time, cmd_data_values in cmd_data.items():
456 457 458 459 460 461 462 463
            entry_data_plotfriendly[cmd]['datetime'].append(cmd_data_time)
            for cmd_data_key, cmd_data_value in cmd_data_values.items():
                entry_data_plotfriendly[cmd][cmd_data_key].append(cmd_data_value)

                plot_filter_data['commands'][cmd][cmd_data_key] += cmd_data_value
                plot_filter_data['commands'][cmd]['number_of_values'] += 1
                plot_filter_data[cmd_data_key] += cmd_data_value
                plot_filter_data['number_of_values'] += 1
Julius Metz's avatar
Julius Metz committed
464 465 466 467
    print(
        'data dict/ filter_data_dict build took {:.1f}s'.format(
            time.time() - dictbuild_starttime),
    )
468
    return entry_data_plotfriendly, plot_filter_data
Julius Metz's avatar
Julius Metz committed
469

Julius Metz's avatar
Julius Metz committed
470

471 472 473 474 475 476 477 478 479 480 481 482
def make_relative_xaxi(all_values):
    """build lists with relative values for xaxis

    Arguments:
        all_values {[datetime, ..]} -- list of all datetime objects for plot

    Returns:
        (list, list) -- 1.: list with the relative values that are displayed on the xaxis
                        2.: list with the datetime as str of the absolute values of xaxis
    """
    min_value = min(all_values)
    max_value = max(all_values)
483 484

    # Is the reference value where the next possible value is searched. For a good number of ticks on the axis.
485
    guideline_tickcounts = 10
486

487 488 489 490
    end_value = (max_value - min_value).total_seconds()
    conversion_factor = 1

    for current_unit in AXIS_INTERVAL_REL:
491
        #  Convert to higher units until the correct one is sufficient
492 493 494 495 496
        if 'max' in current_unit and end_value > current_unit['max']:
            end_value /= current_unit['max']
            conversion_factor *= current_unit['max']
            continue

497
        # find out the best possible interval
498 499 500 501 502
        distances = [
            max(end_value // interval - guideline_tickcounts, guideline_tickcounts - end_value // interval) \
                for interval in current_unit['intervals']
        ]
        interval = current_unit['intervals'][distances.index(min(distances))]
503 504

        # make array with plotly axis data
505 506 507 508 509 510 511 512
        xaxis_ticks = [interval * i for i in range(int(end_value/interval)+1)]
        if len(xaxis_ticks) == 1:
            xaxis_ticks.append(1)
        xaxis_ticks_values = [str(min_value + datetime.timedelta(seconds=tick * conversion_factor)) for tick in xaxis_ticks]
        return xaxis_ticks, xaxis_ticks_values, current_unit['unit']


def make_plot(cmds_data, filter_info, cmd_color, plot_settings, plotly_format_vars, needed_key, relative_xaxis):
Julius Metz's avatar
WC  
Julius Metz committed
513
    """Build a list of dicts in Plotlyconf style for the diffrent traces with the data from cmds_data of needed_key
514
        than make plot dict in Plotly style and change '{host}' in titels.
Julius Metz's avatar
Julius Metz committed
515 516 517 518 519

    Arguments:
        cmds_data {dict} -- is the data for the plot example: {'Command': {'datetime':[...], 'cpu': [...]}}
        filter_info {dict} -- has the information which cmds will be shown example: {'cpu': [cmds]}
        cmd_color {dict} -- assign a fixed plot color to each cmd example:  {'Command': 'rgb(0, 0, 128)'}
Julius Metz's avatar
WC  
Julius Metz committed
520 521
        plot_settings {dict} -- settings for Plotly
        plotly_format_vars {dict} -- values for Plotly settings
Julius Metz's avatar
Julius Metz committed
522
        needed_key {str} -- key of cmds_data for the values to be use
523
        relative_xaxis {bool} -- if true add buttons to change xaxis to relative
Julius Metz's avatar
Julius Metz committed
524 525

    Returns:
526
        dict -- with plotly jsons
Julius Metz's avatar
Julius Metz committed
527 528
    """
    plot_data = []
Julius Metz's avatar
Julius Metz committed
529
    # make plotly plots config
Julius Metz's avatar
Julius Metz committed
530
    for cmd in filter_info[needed_key]:
Julius Metz's avatar
Julius Metz committed
531 532 533
        plot_data.append({
            'x': [str(date) for date in cmds_data[cmd]['datetime']],
            'y': cmds_data[cmd][needed_key],
Julius Metz's avatar
Julius Metz committed
534
            'name': cmd,
Julius Metz's avatar
Julius Metz committed
535
            'marker': {
Julius Metz's avatar
Julius Metz committed
536 537
                'color': cmd_color[cmd],
            },
Julius Metz's avatar
Julius Metz committed
538
            **plot_settings['data'],
Julius Metz's avatar
WC  
Julius Metz committed
539 540 541 542
        }
        )


543 544 545
    layout = pickle.loads(pickle.dumps(plot_settings.get('layout', {}), -1))

    # replace {host} in titles
Julius Metz's avatar
Julius Metz committed
546 547 548
    if 'title' in layout:
        if type(layout['title']) == str:
            layout['title'] = layout['title'].format(
Julius Metz's avatar
Julius Metz committed
549 550 551
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
552
            layout['title']['text'] = layout['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
553 554
                **plotly_format_vars,
            )
Julius Metz's avatar
WC  
Julius Metz committed
555

Julius Metz's avatar
Julius Metz committed
556 557 558
    if 'yaxis' in layout and 'title' in layout['yaxis']:
        if type(layout['yaxis']['title']) == str:
            layout['yaxis']['title'] = layout['yaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
559 560 561
                **plotly_format_vars,
            )
        else:
Julius Metz's avatar
Julius Metz committed
562
            layout['yaxis']['title']['text'] = layout['yaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
563 564
                **plotly_format_vars,
            )
565
    xtitle = ''
Julius Metz's avatar
Julius Metz committed
566 567 568
    if 'xaxis' in layout and 'title' in layout['xaxis']:
        if type(layout['xaxis']['title']) == str:
            layout['xaxis']['title'] = layout['xaxis']['title'].format(
Julius Metz's avatar
Julius Metz committed
569 570
                **plotly_format_vars,
            )
571
            xtitle = layout['xaxis']['title']
Julius Metz's avatar
Julius Metz committed
572
        else:
Julius Metz's avatar
Julius Metz committed
573
            layout['xaxis']['title']['text'] = layout['xaxis']['title']['text'].format(
Julius Metz's avatar
Julius Metz committed
574 575
                **plotly_format_vars,
            )
576
            xtitle = layout['xaxis']['title']['text']
Julius Metz's avatar
WC  
Julius Metz committed
577

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
    # add buttons for switching xaxis if 'relative-absolute_xaxis' in PLOT_CONF is set.
    if relative_xaxis:
            ticks_name, ticks_value, unit = make_relative_xaxi(
                [date for cmd in filter_info[needed_key] for date in cmds_data[cmd]['datetime']],
            )
            if not 'xaxis' in layout:
                layout['xaxis'] = {}
            if not 'updatemenus' in layout:
                layout['updatemenus'] = []
            layout['updatemenus'].append({
                'type': 'buttons',
                'x': 0.6,
                'y': 1.15,
                'direction': 'left',
                'buttons': [
                    {'args':[{'xaxis':{'title': xtitle, 'tickvals': None}}],
                    'label':'absolute xaxis',
                    'method':'relayout'},
                    {'args':[{'xaxis': {'title': 'runtime in {}'.format(unit), 'tickvals': ticks_value, 'ticktext': ticks_name}}],
                    'label':'relative xaxis',
                    'method':'relayout'},
                ],
                'showactive':True,
            })

    return {'data': json.dumps(plot_data), 'layout': json.dumps(layout), 'config': json.dumps(plot_settings.get('config', {}))}
Julius Metz's avatar
Julius Metz committed
604 605 606


def build_html(plots_dict):
Julius Metz's avatar
Julius Metz committed
607 608 609 610 611 612 613 614
    """build html site with the plots in plotsdict

    Arguments:
        plots_dict {dict} -- key: name of plot value: [plot, ...]

    Returns:
        str -- html website
    """
615 616 617 618
    doc, tag, text = Doc().tagtext()
    doc.asis('<!DOCTYPE html>')
    with tag('html'):
        with tag('head'):
Julius Metz's avatar
Julius Metz committed
619 620
            with tag('script', src='plotly.js'):
                pass
621
            # build plotly javascipt in html
Julius Metz's avatar
Julius Metz committed
622 623 624 625
            with tag('script'):
                doc.asis("document.onreadystatechange = () => {if (document.readyState === 'complete') {")
                for name, plots in plots_dict.items():
                    for i, plot in enumerate(plots):
626 627
                        doc.asis("Plotly.newPlot(document.getElementById('{name}-plot-{number}'), JSON.parse('{data}'), JSON.parse('{layout}'), JSON.parse('{config}'));"\
                            .format(name=name, number=i, data=plot['data'], layout=plot['layout'], config=plot['config'])
Julius Metz's avatar
Julius Metz committed
628 629
                        )
                doc.asis('}}')
630 631 632 633 634
        with tag('body'):
            with tag('div', id='index'):
                with tag('h2'):
                    text('Index:')
                with tag('ul'):
Julius Metz's avatar
Julius Metz committed
635
                    for name in plots_dict:
636 637 638
                        with tag('li'):
                            with tag('a', href='#'+name):
                                text(name)
639
            # add div for all plots in html
640 641
            for name, plots in plots_dict.items():
                with tag('div', id=name):
Julius Metz's avatar
Julius Metz committed
642 643 644
                    for i in range(len(plots)):
                        with tag('div', id='{}-plot-{}'.format(name, i)):
                            pass
645 646 647 648

    return doc.getvalue()


Julius Metz's avatar
Julius Metz committed
649 650 651 652 653 654 655 656 657 658 659 660 661
def get_sources(sources):
    """collect all sources (.raw.gz files)

    Arguments:
        sources {tuple} -- tuple of files/directorys

    Returns:
        list -- list of all .raw.gz files which was found
    """
    source_paths = []
    for source in sources:
        source_path =  Path(source)
        if source_path.is_dir():
Julius Metz's avatar
Julius Metz committed
662
            source_paths.extend(source_path.glob('*.raw.gz'))
Julius Metz's avatar
Julius Metz committed
663 664 665 666 667
        elif source_path.is_file() and source.endswith('.raw.gz'):
            source_paths.append(source_path)
    return source_paths


Julius Metz's avatar
Julius Metz committed
668
def data_from_file(arguments):
Julius Metz's avatar
Julius Metz committed
669 670 671 672 673 674 675 676 677 678
    """make data and filter infos of one collectl file

    Arguments:
        arguments {tuple} -- is a tuple of arguments for the function:
                    path {Path} -- collectldata file path
                    collectl {str} -- collectl command
                    shorten_cmds {bool} -- para for parse_file
                    coarsest {bool} -- para for parse_file
                    filtercmds {bool} -- if True cmds will be filter else not
                    filtervalue {int} -- para for filter
Matthias Lieber's avatar
Matthias Lieber committed
679
                    filtertype {str} -- which filter will be called
Julius Metz's avatar
Julius Metz committed
680
                    config {dict} -- plots and merge Config
Julius Metz's avatar
Julius Metz committed
681
    Returns:
Julius Metz's avatar
Julius Metz committed
682 683 684
        (str, dict, dict) -- 1. hostname
                             2. plot_data = {comands : {metrics: [values, ...],  ...}, ...}
                             3. filter_infos = {metrics: [cmds, ...]
Julius Metz's avatar
Julius Metz committed
685
    """
Julius Metz's avatar
Julius Metz committed
686
    path, collectl, shorten_cmds, coarsest, filtercmds, filtervalue, filtertype, config = arguments
Julius Metz's avatar
Julius Metz committed
687
    host = ''
688
    # get hostname
Julius Metz's avatar
Julius Metz committed
689 690 691 692
    with gzip.open(path, 'r') as f:
        for line in f:
            if line.startswith(b'# Host:'):
                host = re.search(r'# Host: *([^ ]+)', line.decode()).group(1)
693
    
Julius Metz's avatar
Julius Metz committed
694
    data, filter_data = parse_file(path, collectl, shorten_cmds, coarsest, config)
Julius Metz's avatar
Julius Metz committed
695 696

    filter_infos = None
Julius Metz's avatar
Julius Metz committed
697 698
    if filtercmds:
        filter_infos = getattr(filter_func, filtertype)(filter_data, filtervalue)
Julius Metz's avatar
Julius Metz committed
699 700 701 702 703 704 705 706 707 708 709 710


    # make new dict with sorted cmd from big to small
    filter_infos_sorted = {}
    for metric in config['NEEDED_VALUES']:
        filter_infos_sorted[metric] = []
        # go over all sorted commands and add if is not filtered out
        for (cmd, _) in sorted(filter_data['commands'].items(), key=lambda x: x[1][metric], reverse=True):
            if not filtercmds or cmd in filter_infos[metric]:
                filter_infos_sorted[metric].append(cmd)

    return host, data, filter_infos_sorted
Julius Metz's avatar
Julius Metz committed
711

712

Julius Metz's avatar
Julius Metz committed
713
@click.command(help='Generate htmlfiles with Plotlyplots with data from collectlfiles(".raw.gz")')
714
@click.option('--source', '-s', multiple=True, default=['.'], show_default=True, type=click.Path(exists=True), help='source for the plots. (collectl data(.raw.gz) file or directory with collectl data) multiple useable')
Julius Metz's avatar
Julius Metz committed
715
@click.option('--collectl', '-c', default='collectl', show_default=True, help='collectl command')
Julius Metz's avatar
Julius Metz committed
716
@click.option('--plotlypath', '-p', type=click.Path(exists=True), help='path to plotly.js')
Julius Metz's avatar
Julius Metz committed
717
@click.option('--destination', '-d', default='.', type=click.Path(exists=True), show_default=True, help='path to directory where directory with plots will be created')
Julius Metz's avatar
Julius Metz committed
718
@click.option('--configpath', default=None, type=click.Path(exists=True), help='python file with plot and merge infos see doku for detail')
Julius Metz's avatar
Julius Metz committed
719
@click.option('--shorten/--notshorten', default=True, help='commands will be shorted only to name')
720
@click.option('--coarsest', is_flag=True, help='commands will be shorted only to type (bash, perl, ...)')
Julius Metz's avatar
Julius Metz committed
721
@click.option('--filtercmds/--notfiltercmds', default=True, help='filtering or not')
Matthias Lieber's avatar
Matthias Lieber committed
722
@click.option('--filtervalue', help='Parameter which is given to the filter.')
Julius Metz's avatar
Julius Metz committed
723 724 725 726 727
@click.option('--filtertype',
              type=click.Choice(FILTER_FUNCTIONS.keys(), case_sensitive=False),
              default=DEFAULT_FILTER, show_default=True,
              help='Filter which is to be used.',
              )
Julius Metz's avatar
Julius Metz committed
728 729 730
@click.option('--force', is_flag=True, help='override existing plot directory if exist')
def main(source, collectl, plotlypath, destination, configpath,
         shorten, coarsest, filtercmds, filtervalue, filtertype, force):
Julius Metz's avatar
Julius Metz committed
731
    source_paths = get_sources(source)
732

Julius Metz's avatar
Julius Metz committed
733
    if not source_paths:
Julius Metz's avatar
Julius Metz committed
734 735 736 737 738 739 740 741 742
        print('no valid source found')
        exit(1)

    if not Path(destination).is_dir():
        print('destination is no valid directory')
        exit(1)

    plots_dir = Path(destination, 'collectlplots')
    if plots_dir.is_dir():
Julius Metz's avatar
Julius Metz committed
743 744 745 746 747 748
        if force:
            shutil.rmtree(plots_dir)
        else:
            print('in destination "collectlplots" already exist')
            exit(1)

749
    # load custome conf if given
Julius Metz's avatar
Julius Metz committed
750 751 752
    config_module = default_plot_conf
    if configpath:
        if configpath.endswith('.py'):
753
            config_module = importlib.machinery.SourceFileLoader('config', configpath).load_module()
Julius Metz's avatar
Julius Metz committed
754 755 756 757 758 759
        else:
            print('given config isn`t a ".py" Python file')
            exit(1)

    # validate config and make dict from values
    config = {}
Julius Metz's avatar
Julius Metz committed
760
    for config_var_name, validator_dict in VALIDATIONS_INFOS.items():
Julius Metz's avatar
Julius Metz committed
761 762 763 764 765
        try:
            config[config_var_name] = getattr(config_module, config_var_name)
        except AttributeError:
            print('{} missing in config'.format(config_var_name))
            exit(1)
Julius Metz's avatar
Julius Metz committed
766 767 768 769
        valid, errormsg = check_valid(config[config_var_name], validator_dict)
        if not valid:
            print('In {0}: \n{1}'.format(config_var_name, errormsg))
            exit(1)
770

Julius Metz's avatar
Julius Metz committed
771 772 773
    data_colllect_functions = []
    for source_path in source_paths:
        data_colllect_functions.append((source_path,
Julius Metz's avatar
Julius Metz committed
774 775 776 777 778 779
                                        collectl,
                                        shorten,
                                        coarsest,
                                        filtercmds,
                                        filtervalue,
                                        FILTER_FUNCTIONS[filtertype],
Julius Metz's avatar
Julius Metz committed
780
                                        config,
Julius Metz's avatar
Julius Metz committed
781
                                        ))
782

Matthias Lieber's avatar
Matthias Lieber committed
783
    # use multiprocessing to parse all collectl output files independently from each other in parallel
Julius Metz's avatar
Julius Metz committed
784 785 786 787
    pool = mp.Pool(min(len(data_colllect_functions), mp.cpu_count()))
    results = pool.map(data_from_file, data_colllect_functions)
    pool.close()
    hosts_data = {}
Julius Metz's avatar
Julius Metz committed
788
    cmd_all = set()
Julius Metz's avatar
Julius Metz committed
789

Julius Metz's avatar
Julius Metz committed
790 791
    for host, data, filter_infos in results:
        hosts_data[host] = {'data': data, 'filter_infos': filter_infos}
Julius Metz's avatar
Julius Metz committed
792
        cmd_all.update([cmd for cmds in filter_infos.values() for cmd in cmds])
Julius Metz's avatar
Julius Metz committed
793

Julius Metz's avatar
Julius Metz committed
794 795
    cmd_colors = {cmd: config['PLOTLY_COLORS'][i % len(config['PLOTLY_COLORS'])] for i, cmd in enumerate(sorted(list(cmd_all)))}
    cmd_colors.update(config['PLOTLY_STATIC_COLORS'])
Julius Metz's avatar
Julius Metz committed
796

Matthias Lieber's avatar
Matthias Lieber committed
797
    # for each host and each plot (as given in config) call make_plot to create html div with plotly
Julius Metz's avatar
Julius Metz committed
798 799 800
    start_plots_build = time.time()
    plots_dict = {}
    for host, host_data in hosts_data.items():
Julius Metz's avatar
Julius Metz committed
801
        plots_dict[host] = {plot_config['name']: [] for plot_config in config['PLOTS_CONFIG']}
Julius Metz's avatar
Julius Metz committed
802
        for plot_config in config['PLOTS_CONFIG']:
Matthias Lieber's avatar
Matthias Lieber committed
803
            plots_dict[host][plot_config['name']].append( make_plot(
Julius Metz's avatar
Julius Metz committed
804 805 806 807 808 809
                host_data['data'],
                host_data['filter_infos'],
                cmd_colors,
                plot_config['plotly_settings'],
                {'host': host},
                plot_config['needed_key'],
810
                plot_config.get('relative-absolute_xaxis', False),
Julius Metz's avatar
Julius Metz committed
811 812 813
            ))
    print("plots build in {:.1f}s".format(time.time() - start_plots_build))

Julius Metz's avatar
Julius Metz committed
814

Matthias Lieber's avatar
Matthias Lieber committed
815
    # create output directory and copy plotly.js
Julius Metz's avatar
Julius Metz committed
816
    plots_dir.mkdir()
Julius Metz's avatar
Julius Metz committed
817 818
    if plotlypath is None:
        plotlypath = Path(__file__).with_name('plotly-latest.min.js')
Julius Metz's avatar
Julius Metz committed
819
    try:
Julius Metz's avatar
Julius Metz committed
820
        shutil.copy(plotlypath, str(Path(plots_dir, 'plotly.js')))
Julius Metz's avatar
Julius Metz committed
821 822 823
    except shutil.SameFileError:
        pass

Matthias Lieber's avatar
Matthias Lieber committed
824
    # for each host create html file with plots
Julius Metz's avatar
Julius Metz committed
825 826
    time_sites = time.time()
    for host, site_plots in plots_dict.items():
Julius Metz's avatar
Julius Metz committed
827 828
        with Path(plots_dir, host+'-plots.html').open(mode='w') as plots_file:
            plots_file.write(build_html(site_plots))
Julius Metz's avatar
Julius Metz committed
829
    print("write/build of websites took {:.1f}s".format(time.time() - time_sites))
Julius Metz's avatar
Julius Metz committed
830 831 832 833


if __name__ == '__main__':
    main()