#!/bin/bash

LOCKPFX="/stage/locks/"
if [[ $(hostname -s) == *-itb ]]; then
    LOCKPFX=${LOCKPFX}itb_
fi

## maximum lock file age (24 hours)
max_lock_age=86400

## warning for lock file age (6 hours)
warn_lock_age=21600

## maximum number of total jobs (to check for insane user submit script)
max_total_jobs=50

# calculate status details
now=`date +%s`
overall_status=0

set_status()
{
    # do not decrease status level
    if [ "$1" -gt $overall_status ]; then
        overall_status=$1
    fi
}

## condor process must exist
condor_ps=`ps -e | grep condor_master | awk '{print $4}'`
if [ $condor_ps ]; then
    condor_status="yes"
else
    condor_status="no"
    set_status 2
fi

# look for old lock file age
oldest_lock_age=-1
lockf=oasis_update_requested_vo
if [ -f ${LOCKPFX}$lockf ]; then
    update_time=`ls -la --time-style +%s ${LOCKPFX}$lockf | awk '{print $6}'`
    oldest_lock_age=`expr $now - $update_time`
fi

if [ $oldest_lock_age -ge $max_lock_age ] ; then
## go to critical
    set_status 2
elif [ $oldest_lock_age -ge $warn_lock_age ] ; then
## go to warning
    set_status 1
fi

total_jobs="`condor_q 2>/dev/null|sed -n 's/^\([0-9]*\) jobs;.*/\1/p'`"
if [ -z "$total_jobs" ]; then
    total_jobs=0
fi
if [ $total_jobs -gt $max_total_jobs ]; then
## go critical if the number is huge
    set_status 2
fi

running_jobs="`condor_q 2>/dev/null|sed -n 's/.* jobs;.* \([0-9]*\) running.*/\1/p'`"
if [ -z "$running_jobs" ]; then
    running_jobs=0
fi

kern=`uname -r`
utime=` cat /proc/uptime | awk '{ print $1 }'`

# begin output, overall status first. OK, WARNING, CRITICAL are allowed values
(
if [ $overall_status -eq 0 ]; then
    echo "OK"
elif [ $overall_status -eq 1 ]; then
    echo "WARNING"
elif [ $overall_status -eq 2 ]; then
    echo "CRITICAL"
fi

# begin details output, only timestamp is mandatory

echo "kernel:$kern"
echo "uptime:$utime"
echo "condor_running:$condor_status"
echo "oldest_lock_age:$oldest_lock_age"
echo "total_jobs:$total_jobs"
echo "running_jobs:$running_jobs"
echo "timestamp:$now"
) > /var/www/html/stamp.new
mv /var/www/html/stamp.new /var/www/html/stamp
