Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
C
Collectl2plotly
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
Matthias Lieber
Collectl2plotly
Commits
2c7bff08
Commit
2c7bff08
authored
Feb 24, 2020
by
Julius Metz
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
generate html file with plots for one collectlfile
parent
035549a8
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
213 additions
and
52 deletions
+213
-52
filter_func.py
filter_func.py
+1
-0
plots_generators.py
plots_generators.py
+18
-33
site_generator.py
site_generator.py
+145
-19
value_merger.py
value_merger.py
+49
-0
No files found.
filter_func.py
View file @
2c7bff08
...
...
@@ -24,6 +24,7 @@ def hardvalue(filter_data, filtervalue):
int
(
100
*
(
cmd_data_value
)
/
filter_data
[
cmd_data_key
]),
cmd
,
))
for
key
,
values
in
tmp_sort_data
.
items
():
current_percent
=
0
for
(
percent
,
cmd
)
in
sorted
(
values
,
key
=
lambda
x
:
x
[
0
],
reverse
=
True
):
...
...
plots_generators.py
View file @
2c7bff08
import
plotly
.io
as
pio
import
plotly
import
datetime
def
build_plot
(
plot_data
,
title
=
None
,
xtitle
=
None
,
ytitle
=
None
,
showlegend
=
True
,
**
kwargs
):
def
build_plot
(
plot_data
,
**
kwargs
):
plot
=
{
'data'
:
plot_data
,
'layout'
:
{
'height'
:
kwargs
.
get
(
'height'
,
500
),
'width'
:
kwargs
.
get
(
'width'
,
None
),
'title'
:
{
'text'
:
title
'text'
:
kwargs
.
get
(
'title'
,
None
),
},
'xaxis'
:
{
'title'
:
xtitle
,
'title'
:
kwargs
.
get
(
'xtitle'
,
None
)
,
},
'yaxis'
:
{
'title'
:
ytitle
,
'title'
:
kwargs
.
get
(
'ytitle'
,
None
),
'type'
:
kwargs
.
get
(
'ytype'
,
None
),
},
'showlegend'
:
showlegend
,
'showlegend'
:
kwargs
.
get
(
'showlegend'
,
True
)
,
},
}
return
plot
return
plot
ly
.
offline
.
plot
(
plot
,
include_plotlyjs
=
False
,
output_type
=
'div'
)
def
cpu_plot
(
cmds_data
,
filter_info
,
**
kwargs
):
needed_key
=
'PCT'
def
default_plot
(
cmds_data
,
filter_info
,
plot_settings
,
**
kwargs
):
plot_data
=
[]
needed_key
=
None
if
'needed_key'
in
kwargs
:
needed_key
=
kwargs
[
'needed_key'
]
else
:
raise
Exception
(
'no needed_key in default_plot generater settings'
)
for
cmd
in
filter_info
[
needed_key
]:
plot_data
.
append
(
{
'type'
:
'scatter'
,
'type'
:
'scatter
gl
'
,
'mode'
:
'markers'
,
'x'
:
cmds_data
[
cmd
].
get
(
'datetime'
,
[]),
'y'
:
cmds_data
[
cmd
].
get
(
needed_key
,
[]),
'name'
:
cmd
,
}
)
pio
.
show
(
build_plot
(
plot_data
,
**
kwargs
))
return
build_plot
(
plot_data
,
**
kwargs
)
def
ram_plot
(
cmds_data
,
filter_info
,
**
kwargs
):
plot_data
=
[]
needed_key
=
'VmRSS'
for
cmd
in
filter_info
[
needed_key
]:
cmd_needed_key_data
=
cmds_data
[
cmd
].
get
(
needed_key
,
[])
plot_data
.
append
(
{
'type'
:
'scatter'
,
'mode'
:
'markers'
,
'x'
:
cmds_data
[
cmd
].
get
(
'datetime'
,
[]),
'y'
:
[
value
/
1024
/
1024
for
value
in
cmd_needed_key_data
],
'name'
:
cmd
,
}
)
pio
.
show
(
build_plot
(
plot_data
,
**
kwargs
))
return
build_plot
(
plot_data
,
**
kwargs
)
return
build_plot
(
plot_data
,
**
plot_settings
)
site_generator.py
View file @
2c7bff08
...
...
@@ -2,12 +2,18 @@ import re
import
subprocess
import
shlex
from
pathlib
import
Path
import
copy
import
time
import
datetime
import
click
import
plotly
from
yattag
import
Doc
import
plots_generators
import
filter_func
import
value_merger
FILTER_FUNCTIONS
=
[
'hardvalue'
,
'average'
]
...
...
@@ -20,28 +26,74 @@ NAME_SPEZIAL_PARAMETER_CONFIG = {
'python'
:
[
'-m'
,
'-W'
,
'-X'
,
'--check-hash-based-pycs'
,
'-c'
],
}
MERGE_VALUES
=
[
'PCT'
,
'VmRSS'
]
NEEDED_VALUES_DEFAULTS
=
{
'default_value'
:
0
,
'default_merger'
:
'addition_int'
,
}
NEEDED_VALUES
=
{
'PCT'
:
{
'keys'
:[
'PCT'
]},
'VmRSS'
:
{
'keys'
:[
'VmRSS'
],
'merger'
:
'x2oneaddition_float_sizedown2'
},
'RKBC'
:
{
'keys'
:[
'RKBC'
],
'merger'
:
'x2oneaddition_float_sizedown1'
},
'WKBC'
:
{
'keys'
:[
'WKBC'
],
'merger'
:
'x2oneaddition_float_sizedown1'
},
'syscalls'
:
{
'keys'
:[
'RSYS'
,
'WSYS'
],
'merger'
:
'x2oneaddtion_int'
},
}
COMAND_BLACKLIST_REGEX
=
[
r'^[^ ]+perl .+collectl'
,
]
PLOT_CONFIG
=
[{
'generator'
:
'cpu_plot'
,
'settings'
:
{
'generator'
:
'default_plot'
,
'name'
:
'cpu'
,
'generator_settings'
:
{
'needed_key'
:
'PCT'
},
'plotly_settings'
:
{
'title'
:
'CPU load'
,
'xtitle'
:
'Date'
,
'ytitle'
:
'CPU load'
,
},
},
{
'generator'
:
'ram_plot'
,
'settings'
:
{
'generator'
:
'default_plot'
,
'name'
:
'ram'
,
'generator_settings'
:
{
'needed_key'
:
'VmRSS'
},
'plotly_settings'
:
{
'title'
:
'Memory Usage'
,
'xtitle'
:
'Date'
,
'ytitle'
:
'RAM usage GiB'
,
},
},
{
'generator'
:
'default_plot'
,
'name'
:
'ior'
,
'generator_settings'
:
{
'needed_key'
:
'RKBC'
},
'plotly_settings'
:
{
'title'
:
'read io'
,
'xtitle'
:
'Date'
,
'ytitle'
:
'I/O MiB/s'
,
'ytype'
:
'log'
,
},
},
{
'generator'
:
'default_plot'
,
'name'
:
'iow'
,
'generator_settings'
:
{
'needed_key'
:
'WKBC'
},
'plotly_settings'
:
{
'title'
:
'write io'
,
'xtitle'
:
'Date'
,
'ytitle'
:
'I/O MiB/s'
,
'ytype'
:
'log'
,
},
},
{
'generator'
:
'default_plot'
,
'name'
:
'ios'
,
'generator_settings'
:
{
'needed_key'
:
'syscalls'
},
'plotly_settings'
:
{
'title'
:
'I/O syscalls'
,
'xtitle'
:
'Date'
,
'ytitle'
:
'I/O syscalls/s'
,
'ytype'
:
'log'
,
},
},
]
...
...
@@ -97,9 +149,12 @@ def get_cmdname(cmd, coarsest=False):
def
parse_file
(
path
,
collectl
,
merge
,
coarsest
):
collectl_starttime
=
time
.
time
()
process
=
subprocess
.
run
(
[
collectl
,
'-P'
,
'-p'
,
path
,
'-sZ'
],
capture_output
=
True
,
)
print
(
'collectl make table took {:.1f}s'
.
format
(
time
.
time
()
-
collectl_starttime
))
parsing_starttime
=
time
.
time
()
output
=
process
.
stdout
.
decode
().
splitlines
()
head
=
output
.
pop
(
0
).
split
(
' '
)
for
possible_head
in
output
[:]:
...
...
@@ -112,6 +167,11 @@ def parse_file(path, collectl, merge, coarsest):
head
[
0
]
=
head
[
0
][
1
:]
head_indexes_dict
=
{
head_title
:
index
for
index
,
head_title
in
enumerate
(
head
)}
entrys_data
=
{}
empty_dict_with_value_titles
=
{
value_title
:
copy
.
deepcopy
(
value_title_settings
.
get
(
'default'
,
NEEDED_VALUES_DEFAULTS
[
'default_value'
])
)
for
value_title
,
value_title_settings
in
NEEDED_VALUES
.
items
()
}
for
entry
in
output
:
splited_entry
=
entry
.
split
(
' '
,
len
(
head_indexes_dict
)
-
1
)
cmd
=
splited_entry
[
-
1
]
...
...
@@ -128,20 +188,27 @@ def parse_file(path, collectl, merge, coarsest):
datetime
.
time
.
fromisoformat
(
splited_entry
[
head_indexes_dict
[
'Time'
]]),
)
if
not
tmp_datetime
in
entrys_data
[
cmd
]:
entrys_data
[
cmd
][
tmp_datetime
]
=
{
key
:
0.0
for
key
in
MERGE_VALUES
}
for
head_title
in
MERGE_VALUES
:
entrys_data
[
cmd
][
tmp_datetime
][
head_title
]
+=
float
(
splited_entry
[
head_indexes_dict
[
head_title
]])
entrys_data
[
cmd
][
tmp_datetime
]
=
copy
.
deepcopy
(
empty_dict_with_value_titles
)
for
value_title
,
value_title_settings
in
NEEDED_VALUES
.
items
():
entrys_data
[
cmd
][
tmp_datetime
][
value_title
]
=
getattr
(
value_merger
,
value_title_settings
.
get
(
'merger'
,
NEEDED_VALUES_DEFAULTS
[
'default_merger'
]),
)(
entrys_data
[
cmd
][
tmp_datetime
][
value_title
],
*
[
splited_entry
[
head_indexes_dict
[
key
]]
for
key
in
value_title_settings
[
'keys'
]],
)
#float(splited_entry[head_indexes_dict[head_title]])
print
(
'parsing/merge took {:.1f}s'
.
format
(
time
.
time
()
-
parsing_starttime
))
dictbuild_starttime
=
time
.
time
()
entry_data_plotfriendly
=
{}
plot_filter_data
=
{
key
:
0.0
for
key
in
MERGE_VALUES
}
plot_filter_data
=
copy
.
deepcopy
(
empty_dict_with_value_titles
)
plot_filter_data
[
'number_of_values'
]
=
0
plot_filter_data
[
'commands'
]
=
{}
for
cmd
,
cmd_data
in
entrys_data
.
items
():
plot_filter_data
[
'commands'
][
cmd
]
=
{
key
:
0
for
key
in
MERGE_VALUES
}
plot_filter_data
[
'commands'
][
cmd
]
=
copy
.
deepcopy
(
empty_dict_with_value_titles
)
plot_filter_data
[
'commands'
][
cmd
][
'number_of_values'
]
=
0
entry_data_plotfriendly
[
cmd
]
=
{
key
:
[]
for
key
in
MERGE_VALUES
}
entry_data_plotfriendly
[
cmd
]
=
{
key
:
[]
for
key
in
NEEDED_VALUES
.
keys
()
}
entry_data_plotfriendly
[
cmd
][
'datetime'
]
=
[]
for
cmd_data_time
,
cmd_data_values
in
cmd_data
.
items
():
entry_data_plotfriendly
[
cmd
][
'datetime'
].
append
(
cmd_data_time
)
...
...
@@ -152,30 +219,89 @@ def parse_file(path, collectl, merge, coarsest):
plot_filter_data
[
'commands'
][
cmd
][
'number_of_values'
]
+=
1
plot_filter_data
[
cmd_data_key
]
+=
cmd_data_value
plot_filter_data
[
'number_of_values'
]
+=
1
print
(
'data dict/ filter_data_dict build took {:.1f}s'
.
format
(
time
.
time
()
-
dictbuild_starttime
))
return
entry_data_plotfriendly
,
plot_filter_data
def
build_html
(
plots_dict
,
plotlypath
):
doc
,
tag
,
text
=
Doc
().
tagtext
()
doc
.
asis
(
'<!DOCTYPE html>'
)
with
tag
(
'html'
):
with
tag
(
'head'
):
with
tag
(
'script'
):
with
Path
(
plotlypath
).
open
(
mode
=
'r'
)
as
f
:
doc
.
asis
(
f
.
read
())
with
tag
(
'script'
):
doc
.
asis
(
'''toggleplot = (name, index) => {
const elem = document.querySelector('#'+name+' div:nth-child('+index+')');
if (window.getComputedStyle(elem).display === "none") {
elem.style.display = 'contents';
} else {
elem.style.display = 'none';
}
}'''
)
with
tag
(
'body'
):
with
tag
(
'div'
,
id
=
'index'
):
with
tag
(
'h2'
):
text
(
'Index:'
)
with
tag
(
'ul'
):
for
name
in
plots_dict
.
keys
():
with
tag
(
'li'
):
with
tag
(
'a'
,
href
=
'#'
+
name
):
text
(
name
)
for
name
,
plots
in
plots_dict
.
items
():
with
tag
(
'div'
,
id
=
name
):
with
tag
(
'h2'
):
text
(
name
)
for
i
,
plot
in
enumerate
(
plots
):
with
tag
(
'button'
,
onclick
=
'toggleplot("{}", {})'
.
format
(
name
,
i
+
1
)):
doc
.
asis
(
'toggle host'
)
doc
.
asis
(
plot
)
return
doc
.
getvalue
()
@
click
.
command
()
@
click
.
option
(
'--file'
,
'-f'
,
required
=
True
)
@
click
.
option
(
'--collectl'
,
'-c'
,
required
=
False
,
default
=
'collectl'
)
@
click
.
option
(
'--plotlypath'
,
'-p'
,
required
=
True
)
@
click
.
option
(
'--destination'
,
'-d'
,
required
=
False
,
default
=
'.'
)
@
click
.
option
(
'--merge/--notmerge'
,
default
=
True
)
@
click
.
option
(
'--coarsest/--notcoarsest'
,
default
=
False
)
@
click
.
option
(
'--filtercmd
/--notfiltercmd
'
,
default
=
True
)
@
click
.
option
(
'--filtercmd
s/--notfiltercmds
'
,
default
=
True
)
@
click
.
option
(
'--filtervalue'
,
'-v'
,
type
=
int
,
default
=
90
)
@
click
.
option
(
'--filtertype'
,
'-t'
,
type
=
click
.
Choice
(
FILTER_FUNCTIONS
,
case_sensitive
=
False
),
default
=
FILTER_FUNCTIONS
[
0
])
def
main
(
file
,
collectl
,
merge
,
coarsest
,
filtercmd
,
filtervalue
,
filtertype
):
def
main
(
file
,
collectl
,
plotlypath
,
destination
,
merge
,
coarsest
,
filtercmds
,
filtervalue
,
filtertype
):
path
=
Path
(
file
)
if
path
.
exists
():
data
,
filter_data
=
parse_file
(
path
,
collectl
,
merge
,
coarsest
)
if
filtercmd
:
if
filtercmd
s
:
filter_infos
=
getattr
(
filter_func
,
filtertype
)(
filter_data
,
filtervalue
)
else
:
filter_infos
=
{
key
:
list
(
data
.
keys
())
for
key
in
NEEDED_VALUES
.
keys
()}
plots_dict
=
{
config
[
'name'
]:
[]
for
config
in
PLOT_CONFIG
}
for
plot_config
in
PLOT_CONFIG
:
plot
=
getattr
(
plots_generators
,
plot_config
[
'generator'
])(
data
,
filter_infos
,
**
plot_config
[
'settings'
])
plots_dict
[
plot_config
[
'name'
]].
append
(
getattr
(
plots_generators
,
plot_config
[
'generator'
])(
data
,
filter_infos
,
plot_config
[
'plotly_settings'
],
**
plot_config
[
'generator_settings'
],
))
with
Path
(
destination
,
'plots.html'
).
open
(
mode
=
'w'
)
as
f
:
f
.
write
(
build_html
(
plots_dict
,
plotlypath
))
...
...
value_merger.py
0 → 100644
View file @
2c7bff08
def
addition_int
(
old_v
,
to_merge_v
):
"""sum old_v and int(to_merge_v)
Arguments:
old_v {int} -- first value
to_merge_v {str/int/float} -- value to add to old_v
Returns:
int -- sum of old_v and intto_merge_v
"""
return
old_v
+
int
(
to_merge_v
)
def
x2oneaddtion_int
(
old_v
,
*
args
):
"""sum x values to old_v
Arguments:
old_v {int} -- first value
Returns:
int -- sum x values to old_v
"""
return
old_v
+
sum
([
int
(
x
)
for
x
in
args
])
def
x2oneaddition_float_sizedown1
(
old_v
,
*
args
):
"""sum x values / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 1024
"""
return
old_v
+
(
sum
([
float
(
x
)
for
x
in
args
])
/
1024
)
def
x2oneaddition_float_sizedown2
(
old_v
,
*
args
):
"""sum x values / 1024 / 1024 to old_v
Arguments:
old_v {float} -- first value
Returns:
int -- sum x values to old_v / 1024 / 1024
"""
return
old_v
+
(
sum
([
float
(
x
)
for
x
in
args
])
/
1024
/
1024
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment