pika_post_processing.sh 7.09 KB
Newer Older
fwinkler's avatar
fwinkler committed
1 2 3 4 5 6 7 8 9 10 11 12 13
#!/bin/bash

source /sw/taurus/tools/pika/pika-current.conf
DEBUG_PATH=/tmp/pika_post_proccesing.out
LOCK_POST_PROCESSING_SCRIPT=/tmp/pika_post_proccesing.lock

# get utility functions
source ${PIKA_ROOT}/pika_utils.sh

function create_lock()
{
  # lock script
  if [ -f ${LOCK_POST_PROCESSING_SCRIPT} ]; then
14 15
    echo -e "Post processing is already running..." 2>&1 | tee -a $DEBUG_PATH
    echo -e "exit 0" 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
16 17 18 19 20 21 22 23 24
    exit 0
  else
    touch ${LOCK_POST_PROCESSING_SCRIPT}
    echo -e "Start post processing..." |2>&1 tee $DEBUG_PATH
  fi
}

function release_lock()
{
25 26 27 28 29 30
  #check debug output
  echo "failed" 2>&1 | tee -a $DEBUG_PATH
  echo "halloerrorhallo" 2>&1 | tee -a $DEBUG_PATH
  echo "ModuleNotFoundError" 2>&1 | tee -a $DEBUG_PATH
  check_debug

fwinkler's avatar
fwinkler committed
31 32
  if [ -f ${LOCK_POST_PROCESSING_SCRIPT} ]; then
    rm -f ${LOCK_POST_PROCESSING_SCRIPT}
33
    echo -e "Post processing finished." 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
34 35 36
  fi
}

37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
function check_debug()
{
  #grep for error or failed and send email if necessary
  email_text=/tmp/pika_pp_email.txt
  touch ${email_text}

  egrep -i "error|failed" $DEBUG_PATH >> ${email_text}
  #check size of mail text
  email_text_size=$(stat -c%s "${email_text}")

  if [ ${email_text_size} -gt 0 ]; then
    send_email "[PIKA] post processing errors" ${email_text}
  fi

  rm -f ${email_text}
}

fwinkler's avatar
fwinkler committed
54 55
function backup_influx()
{
56 57
  echo "Backup influxdb..." 2>&1 | tee -a $DEBUG_PATH
  source ${PIKA_ROOT}/post_processing/backup_influxdb.sh 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
58 59 60 61
}

function restore_influx()
{
62
  echo "Restore influxdb..." 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
63 64
  # get all shards which are not restored yet
  restore_shards=`mysql_command "select shard_id, start from Backup where success=1 and restored=0"`
65
  #echo ${restore_shards} 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
66 67 68 69 70 71

  # convert string output into indexed array
  IFS=', ' read -r -a shard_array <<< "${restore_shards}"

  len=${#shard_array[@]}
  if [ ${len} -lt 2 ]; then
72
    echo "Cannot find shards for restoring." 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
73 74 75 76 77 78 79 80 81
    return 0
  fi

  len=$(( ${len} - 2 ))
  i=0
  while [ $i -le ${len} ]
  do
    shard_id=${shard_array[i]}
    start_seconds=${shard_array[i+1]}
82 83

    source ${PIKA_ROOT}/post_processing/restore_influxdb.sh ${shard_id}_${start_seconds} 2>&1 | tee -a $DEBUG_PATH
fwinkler's avatar
fwinkler committed
84 85 86 87 88 89 90 91 92
    if [ $? -eq 0 ]; then
      mysql_command "UPDATE Backup SET restored=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
    fi
    i=$(( $i + 2 ))
  done
}

function revise_metadata()
{
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
  echo "Revise metadata..." 2>&1 | tee -a $DEBUG_PATH
  # get all shards which are not revised yet
  revise_shards=`mysql_command "select shard_id,start,end from Backup where success=1 and restored=1 and metadata_corrected=0"`
  #echo ${revise_shards} 2>&1 | tee -a $DEBUG_PATH

  # convert string output into indexed array
  IFS=', ' read -r -a shard_array <<< "${revise_shards}"

  len=${#shard_array[@]}
  if [ ${len} -lt 3 ]; then
    echo "Cannot find shards for metadata revision." 2>&1 | tee -a $DEBUG_PATH
    return 0
  fi

  len=$(( ${len} - 3 ))
  i=0
  while [ $i -le ${len} ]
  do
    shard_id=${shard_array[i]}
    start_seconds=${shard_array[i+1]}
    end_seconds=${shard_array[i+2]}

    python3 ${PIKA_ROOT}/post_processing/revise_mariadb.py --start=${start_seconds} --end=${end_seconds} 2>&1 | tee -a $DEBUG_PATH
    if [ $? -eq 0 ]; then

      #delete jobs which have corrupt data
      mysql_command "delete from Job_Data where NAME='n/a' and START>${start_seconds} and START<${end_seconds};"
      #update jobs with negative duration or pending time
      mysql_command "update Job_Data set START=END-40 where (END-START)<0 and STATUS<>'running' and START>${start_seconds} and START<${end_seconds};"
      mysql_command "update Job_Data set SUBMIT=START-1 where (START-SUBMIT)<0 and START>${start_seconds} and START<${end_seconds};"

      mysql_command "update Backup set metadata_corrected=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
    fi
    i=$(( $i + 3 ))
  done
fwinkler's avatar
fwinkler committed
128 129 130 131
}

function create_footprints()
{
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
  echo "Create footprints..." 2>&1 | tee -a $DEBUG_PATH
  # get all shards without footprints
  footprint_shards=`mysql_command "select shard_id,start,end from Backup where success=1 and restored=1 and metadata_corrected=1 and footprints_created=0"`
  #echo ${footprint_shards} 2>&1 | tee -a $DEBUG_PATH

  # convert string output into indexed array
  IFS=', ' read -r -a shard_array <<< "${footprint_shards}"

  len=${#shard_array[@]}
  if [ ${len} -lt 3 ]; then
    echo "Cannot find shards for footprint generation." 2>&1 | tee -a $DEBUG_PATH
    return 0
  fi

  len=$(( ${len} - 3 ))
  i=0
  while [ $i -le ${len} ]
  do
    shard_id=${shard_array[i]}
    start_seconds=${shard_array[i+1]}
    #set start one week before to get footprints for overlapping jobs
    if [ $i -eq 0 ]; then
      start_footprint=$((start_seconds-(3600*24*7)))
    else
      start_footprint=${start_seconds}
    fi
    end_seconds=${shard_array[i+2]}

    python3 ${PIKA_ROOT}/post_processing/footprints.py --start=${start_footprint} --end=${end_seconds} --store_to_db 2>&1 | tee -a $DEBUG_PATH
    if [ $? -eq 0 ]; then
      mysql_command "update Backup set footprints_created=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
    fi
    i=$(( $i + 3 ))
  done
fwinkler's avatar
fwinkler committed
166 167 168 169
}

function create_tags()
{
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
  echo "Create tags..." 2>&1 | tee -a $DEBUG_PATH
  # get all shards without tags
  tag_shards=`mysql_command "select shard_id,start,end from Backup where success=1 and restored=1 and metadata_corrected=1 and footprints_created=1 and tags_created=0"`
  #echo ${tag_shards} 2>&1 | tee -a $DEBUG_PATH

  # convert string output into indexed array
  IFS=', ' read -r -a shard_array <<< "${tag_shards}"

  len=${#shard_array[@]}
  if [ ${len} -lt 3 ]; then
    echo "Cannot find shards for tag generation." 2>&1 | tee -a $DEBUG_PATH
    return 0
  fi

  len=$(( ${len} - 3 ))
  i=0
  while [ $i -le ${len} ]
  do
    shard_id=${shard_array[i]}
    start_seconds=${shard_array[i+1]}
    #set start one week before to get tags for overlapping jobs
    if [ $i -eq 0 ]; then
      start_tag=$((start_seconds-(3600*24*7)))
    else
      start_tag=${start_seconds}
    fi
    end_seconds=${shard_array[i+2]}

    python3 ${PIKA_ROOT}/post_processing/tags.py --start=${start_tag} --end=${end_seconds} 2>&1 | tee -a $DEBUG_PATH
    if [ $? -eq 0 ]; then
      mysql_command "update Backup set tags_created=1 WHERE shard_id=${shard_id} AND start=${start_seconds};"
    fi
    i=$(( $i + 3 ))
  done
fwinkler's avatar
fwinkler committed
204 205 206 207 208 209 210 211 212 213 214 215
}

if [ ! $# -eq 1 ]; then
  echo "One argument required!"
  echo "./pika_post_processing [all|backup|restore|revise|footprint|tag]"
else

  case "$1" in
    all)
      create_lock
      backup_influx
      restore_influx
216 217 218
      revise_metadata
      create_footprints
      create_tags
fwinkler's avatar
fwinkler committed
219 220 221 222 223 224 225 226 227 228 229 230
      release_lock
      ;;
    backup)
      create_lock
      backup_influx
      release_lock
      ;;
    restore)
      create_lock
      restore_influx
      release_lock
      ;;
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
    revise)
      create_lock
      revise_metadata
      release_lock
      ;;
    footprint)
      create_lock
      create_footprints
      release_lock
      ;;
    tag)
      create_lock
      create_tags
      release_lock
      ;;
fwinkler's avatar
fwinkler committed
246 247 248 249 250
    *)
      echo "$1 is not supported."
      ;;
  esac
fi