#!/bin/bash

## warning time since all whitelists were successfully replicated (24 hours)
warn_white_age=86400

## number of squid processes that must be running
need_squids=2

## required number of httpd running
need_httpd=4

now=`date +%s`
overall_status=0

set_status()
{
    # do not decrease status level
    if [ "$1" -gt $overall_status ]; then
        overall_status=$1
   fi
}

## check for problems with repositories from cvmfs-servermon
## to see full detailed output see
## http://oasis-replica.opensciencegrid.org/cvmfsmon/api/v1.0/all&format=details
## repositories can be excluded in /etc/cvmfsmon/api.conf

critical_repos=""
warning_repos=""
lines="`wget -qO- http://localhost:8080/cvmfsmon/api/v1.0/all 2>/dev/null`"
if [ -z "$lines" ]; then
    critical_repos="ALL"
    set_status 2
else
    critical_repos="`echo "$lines" | sed -n 's/CRITICAL://p'`"
    warning_repos="`echo "$lines" | sed -n 's/WARNING://p'`"
    if [ -n "$critical_repos" ] || [ -n "$warning_repos" ]; then
	# we set them both as a warning for the GOC since they're
	#  not bad enough to trigger off-hours alarms
	set_status 1
    fi
fi


## determine whitelist age

oldest_white_age=0
whitelist_warning_repos=""
for repo in `cd /etc/cvmfs/repositories.d;echo *.*`; do
    if [[ "$repo" == *.cern.ch ]]; then
        # cern.ch repos are grandfathered, not signed by OSG key
        continue
    fi
    if [ ! -f /srv/cvmfs/$repo/.cvmfspublished ]; then
	# ignore repositories that haven't finished their initial snapshot
	continue
    fi
    whitemodded=`stat -c %Y /oasissrv/cvmfs/$repo/.cvmfswhitelist 2>/dev/null`
    if [ -n "$whitemodded" ]; then
        now=`date +%s`
        white_age=`expr $now - $whitemodded`
    else
        white_age=999999
    fi
    if [ $white_age -gt $oldest_white_age ]; then
        oldest_white_age=$white_age
    fi

    origfingerprint="`sed -n '/^N/{n;p;q}' /srv/cvmfs/$repo/.cvmfswhitelist 2>/dev/null`"
    oasisfingerprint="`sed -n '/^N/{n;p;q}' /oasissrv/cvmfs/$repo/.cvmfswhitelist 2>/dev/null`"

    if [ $white_age -ge $warn_white_age ] || [ "$origfingerprint" != "$oasisfingerprint" ]; then
    ## go to warning
        set_status 1
        if [ -n "$whitelist_warning_repos" ]; then
            whitelist_warning_repos="$whitelist_warning_repos,"
        fi
        whitelist_warning_repos="$whitelist_warning_repos$repo"
    fi
done

## are the squid processes running?
squids=`ps -e | grep -c squid`

squid_status="yes"
if [ $squids -lt $need_squids ] ; then
    set_status 2
    squid_status="no"
fi

## are there httpd processes running?
httpds=`ps -e | grep -c httpd`

httpd_status="yes"
if [ $httpds -lt $need_httpd ] ; then
    set_status 2
    httpd_status="no"
fi

kern=`uname -r`
utime=` cat /proc/uptime | awk '{ print $1 }'`

# begin output, overall status first. OK, WARNING, CRITICAL are allowed values
(
if [ $overall_status -eq 0 ]; then
    echo "OK"
elif [ $overall_status -eq 1 ]; then
    echo "WARNING"
elif [ $overall_status -eq 2 ]; then
    echo "CRITICAL"
fi

# begin details output, only timestamp is mandatory

echo "kernel:$kern" 
echo "uptime:$utime" 
echo "critical_repos:$critical_repos"
echo "warning_repos:$warning_repos"
echo "whitelist_warning_repos:$whitelist_warning_repos"
echo "oldest_whitelist_age:$oldest_white_age"
echo "squid_running:$squid_status"
echo "httpd_running:$httpd_status"
echo "timestamp:$now"
) > /var/www/html/stamp.new
mv /var/www/html/stamp.new /var/www/html/stamp
