Commit ac095a3a authored by fwinkler's avatar fwinkler

Improvements in post processing scripts.

parent b165b407
......@@ -45,11 +45,11 @@ unitthresholds = (
# conversion factor for unit prefixes: binary, decimal
unit2fac = {
"P": (1<<50L, 10**15),
"T": (1<<40L, 10**12),
"G": (1<<30L, 10**9),
"M": (1<<20L, 10**6),
"K": (1<<10L, 10**3),
"P": (1<<50, 10**15),
"T": (1<<40, 10**12),
"G": (1<<30, 10**9),
"M": (1<<20, 10**6),
"K": (1<<10, 10**3),
"": (1, 1)
}
......
......@@ -84,7 +84,8 @@ function restore_influx()
start_seconds=${shard_array[i+1]}
source ${PIKA_ROOT}/post_processing/restore_influxdb.sh ${shard_id}_${start_seconds} 2>&1 | tee -a $DEBUG_PATH
if [ $? -eq 0 ]; then
error_text=`egrep -i "error|failed" $DEBUG_PATH | grep -v "Failed "`
if [ ${#error_text} -eq 0 ]; then
mysql_command "UPDATE Backup SET restored=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
fi
i=$(( $i + 2 ))
......@@ -116,7 +117,8 @@ function revise_metadata()
end_seconds=${shard_array[i+2]}
python3 ${PIKA_ROOT}/post_processing/revise_mariadb.py --start=${start_seconds} --end=${end_seconds} 2>&1 | tee -a $DEBUG_PATH
if [ $? -eq 0 ]; then
error_text=`egrep -i "error|failed" $DEBUG_PATH`
if [ ${#error_text} -eq 0 ]; then
#delete jobs which have corrupt data
mysql_command "delete from Job_Data where NAME='n/a' and START>${start_seconds} and START<${end_seconds};"
......@@ -161,7 +163,8 @@ function create_footprints()
end_seconds=${shard_array[i+2]}
python3 ${PIKA_ROOT}/post_processing/footprints.py --start=${start_footprint} --end=${end_seconds} --store_to_db 2>&1 | tee -a $DEBUG_PATH
if [ $? -eq 0 ]; then
error_text=`egrep -i "error|failed" $DEBUG_PATH`
if [ ${#error_text} -eq 0 ]; then
mysql_command "update Backup set footprints_created=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
fi
i=$(( $i + 3 ))
......@@ -199,7 +202,8 @@ function create_tags()
end_seconds=${shard_array[i+2]}
python3 ${PIKA_ROOT}/post_processing/tags.py --start=${start_tag} --end=${end_seconds} 2>&1 | tee -a $DEBUG_PATH
if [ $? -eq 0 ]; then
error_text=`egrep -i "error|failed" $DEBUG_PATH`
if [ ${#error_text} -eq 0 ]; then
mysql_command "update Backup set tags_created=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
fi
i=$(( $i + 3 ))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment