Commit bc7649a8 authored by Frank Winkler's avatar Frank Winkler
Browse files

Merge branch 'master' of gitlab.hrz.tu-chemnitz.de:pika/packages

parents 70725356 7b581919
# Pika Packages
## Docker Containers for Databases and GUI
There are three Docker containers: `pika-mariadb`, `pika-influxdb` and `pika-gui`.
......@@ -14,7 +16,6 @@ It is recommended to install these three containers in the following order:
2. [pika-influxdb](pika-influxdb)
3. [pika-gui](pika-gui)
***
#### Pika Bridge Network
All containers are installed with the external bridge network `pika-net`.
......@@ -40,4 +41,26 @@ If you install containers on the same host, e.g. `pika-mariadb` and `pika-gui`,
## Performance Data Collection Daemon
The performance data collection daemon must be installed on each compute node.
The Pika Software-Stack on a compute node consists of the following components.
* collectd - We provide a patched version of collectd for Pika.
* likwid - The likwid performance monitoring and benchmark suite.
* collectd-likwid-plugin - Our additional collectd-likwid-plugin to access likwid performance data.
* The pika slurm spank-plugin - Our slurm-plugin to control job-monitoring.
We provide separate RPM packages for all of these plus the `pika.rpm` package, which includes the configuration and slurm prolog/epilog scripts.
## Install RPMs
After building the rpm packages they need to be installed.
Actual package names may differ.
```
rpm -i collectd.rpm
rpm -i likwid.rpm
rpm -i pika-collectd-likwid-plugin-0.1-1.el7.x86_64.rpm
rpm -i pika-spank
rpm -i pika
```
# pika-collectd-likwid-plugin
## Build RPM package
You need to build the `pika-collectd` and `pika-likwid` packages before.
Afterwards, copy the resulting RPM files:
collectd-5.12.0-1.el7.x86_64.rpm
likwid-5.1.0-1.x86_64.rpm
likwid-devel-5.1.0-1.x86_64.rpm
… in this directory. Then, to build the pika-collectd-likwid-plugin RPM just run.
```
./build.sh
```
#! /bin/bash
get_collectd_sources() {
wget -c "https://collectd.org/files/collectd-5.12.0.tar.bz2"
}
build_docker_image() {
if [[ $(docker images -q collectd-likwid-plugin-build) == "" ]]; then
docker build -t collectd-likwid-plugin-build "$(pwd)/../centos-dev-docker/"
fi
}
run_build() {
build_docker_image
docker run --rm --volume="$(pwd)":/src --workdir=/src collectd-likwid-plugin-build collectd.spec
}
check_rpm_deps() {
[[ -f collectd-5.12.0-1.el7.x86_64.rpm ]] && \
[[ -f likwid-5.1.0-1.x86_64.rpm ]] && \
[[ -f likwid-devel-5.1.0-1.x86_64.rpm ]] || echo "Error RPM dependencies not satisfied!"
}
check_rpm_deps || exit 1
if [[ ! -d collectd-5.12.0 ]]; then
get_collectd_sources
tar xjf collectd-5.12.0.tar.bz2
fi
run_build
# pika-collectd
Create the collectd RPM package for Pika, just run.
```sh
./build.sh
```
......@@ -2,7 +2,7 @@
build_docker_image() {
if [[ $(docker images -q rpmbuild) == "" ]]; then
docker build -t rpmbuild "$(pwd)../rpmbuild-docker/"
docker build -t rpmbuild "$(pwd)/../rpmbuild-docker/"
fi
}
......
......@@ -10,8 +10,8 @@
#HostnameReplace
TypesDB "CD_INST_PATH/share/collectd/types.db"
TypesDB "CUSTOM_TYPES_DIR/custom_types.db"
TypesDB "/usr/share/collectd/types.db"
TypesDB "/usr/share/collectd/custom_types.db"
Interval 30
AlignRead true
......
......@@ -14,6 +14,7 @@ Source1: collectd.conf
Source2: ib_bw.py
Source3: influx_write.py
Source4: lustre_bw.py
Source5: custom_types.db
BuildRequires: perl(ExtUtils::MakeMaker)
BuildRequires: perl(ExtUtils::Embed)
......@@ -628,6 +629,7 @@ make install DESTDIR="%{buildroot}"
install -Dp -m0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/collectd.conf
install -Dp -m0644 contrib/systemd.collectd.service %{buildroot}%{_unitdir}/collectd.service
install -Dp -m0644 %{SOURCE5} %{buildroot}%{_datadir}/collectd/custom_types.db
install -d %{buildroot}%{_datadir}/collectd/plugins/
install -Dp -m0644 %{SOURCE2} %{buildroot}%{_datadir}/collectd/plugins/ib_bw.py
install -Dp -m0644 %{SOURCE3} %{buildroot}%{_datadir}/collectd/plugins/influx_write.py
......@@ -684,6 +686,7 @@ make check
%{_sbindir}/collectd
%{_sbindir}/collectdmon
%{_datadir}/collectd/types.db
%{_datadir}/collectd/custom_types.db
%{_datadir}/collectd/plugins/ib_bw.py
%{_datadir}/collectd/plugins/influx_write.py
%{_datadir}/collectd/plugins/lustre_bw.py
......
likwid value:GAUGE:0:U
pair value:GAUGE:0:U, value:GAUGE:0:U
PROJECT_NAME = pika
FILES = README.md \
pika_access_template \
pika.conf \
pika.spec \
slurm/epilog.sh \
slurm/prolog.sh \
slurm/utils/pika_update_metadata_epilog_include.sh \
slurm/utils/pika_utils.sh \
slurm/utils/pika_slurm_env_epilog.sh \
slurm/utils/pika_save_metadata_prolog_include.sh \
slurm/utils/pika_slurm_env_prolog.sh
dist: $(FILES)
rm -rf $(PROJECT_NAME)
mkdir $(PROJECT_NAME)
cp -r $(FILES) $(PROJECT_NAME)
tar czf $(PROJECT_NAME).tar.gz $(PROJECT_NAME)
rpm: dist
rm -rf rpmbuild
mkdir -p rpmbuild/SOURCES
cp $(PROJECT_NAME).tar.gz rpmbuild/SOURCES/.
rpmbuild --define="_topdir `pwd`/rpmbuild" -ba pika.spec
clean:
rm -f $(PROJECT_NAME).tar.gz
rm -rf $(PROJECT_NAME)
rm -rf rpmbuild
## Setup Database Connection
## Build RPM
Make a copy of `pika_access_template` and adjust it accordingly.
To build the rpm package just run `make rpm`. Note that you need to have `rpmbuild` installed.
cp pika_access_template pika_access
## Specify Pika Root
Edit `pika.conf` to specify `PIKA_ROOT`.
Also, make sure that prolog and epilog can find `pika.conf` by adjusting the source command in `slurm/prolog.sh` and `slurm/epilog.sh`.
Name: pika
Summary: Center-Wide and Job-Aware Cluster Monitoring
Version: 1.0.0
Release: 1%{?dist}
License: GPL-3.0
Source: pika.tar.gz
Group: Development/Tools
Packager: Sebastian Oeste <sebastian.oeste@tu-dresden.de>
URL: https://gitlab.hrz.tu-chemnitz.de/pika/monitoring
Requires: collectd
Requires: likwid
Requires: pika-collectd-likwid-plugin
Requires: clustershell
%description
Pika is a software suite for Center-Wide and Job-Aware Cluster Monitoring,
the best you can have since cutted bread.
%prep
%setup -q -n pika
%install
# install configs
install -Dp -m0755 pika.conf %{buildroot}/etc/pika/pika-control/pika.conf
install -Dp -m0755 pika_access_template %{buildroot}/etc/pika/pika-control/pika_access
# install slurm stuff
install -Dp -m0755 epilog.sh %{buildroot}/etc/pika/pika-control/slurm/epilog.sh
install -Dp -m0755 prolog.sh %{buildroot}/etc/pika/pika-control/slurm/prolog.sh
install -Dp -m0755 pika_update_metadata_epilog_include.sh %{buildroot}/etc/pika/pika-control/slurm/utils/pika_update_metadata_epilog_include.sh
install -Dp -m0755 pika_utils.sh %{buildroot}/etc/pika/pika-control/slurm/utils/pika_utils.sh
install -Dp -m0755 pika_slurm_env_epilog.sh %{buildroot}/etc/pika/pika-control/slurm/utils/pika_slurm_env_epilog.sh
install -Dp -m0755 pika_save_metadata_prolog_include.sh %{buildroot}/etc/pika/pika-control/slurm/utils/pika_save_metadata_prolog_include.sh
install -Dp -m0755 pika_slurm_env_prolog.sh %{buildroot}/etc/pika/pika-control/slurm/utils/pika_slurm_env_prolog.sh
%files
#%doc README.md flow_graph.png flow_graph.svg
#%{_datadir}/pika/pika-control/pika.conf
/etc/pika/pika-control/pika.conf
/etc/pika/pika-control/pika_access
/etc/pika/pika-control/slurm/epilog.sh
/etc/pika/pika-control/slurm/prolog.sh
/etc/pika/pika-control/slurm/utils/pika_update_metadata_epilog_include.sh
/etc/pika/pika-control/slurm/utils/pika_utils.sh
/etc/pika/pika-control/slurm/utils/pika_slurm_env_epilog.sh
/etc/pika/pika-control/slurm/utils/pika_save_metadata_prolog_include.sh
/etc/pika/pika-control/slurm/utils/pika_slurm_env_prolog.sh
# pika-likwid
Create the likwid RPM package for Pika, just run.
```sh
./build.sh
```
......@@ -4,7 +4,7 @@ LIKWID_VERSION=5.1.0
build_docker_image() {
if [[ $(docker images -q rpmbuild) == "" ]]; then
docker build -t rpmbuild "$(pwd)../rpmbuild-docker/"
docker build -t rpmbuild "$(pwd)/../rpmbuild-docker/"
fi
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment