#! /bin/bash

# This script is designed to run from a systemd service, which
# in turn is run from a systemd timer.  Therefore any state needs
# to reside in the filesystem.

# It implement system monitoring for filesystem, memory, and cpu use
# and reports issues to syslog

svc=sysmonitor
svcdir=/run/${svc}

# default values, in case config file not present
# or no settings

# this can be set in sysmonitor.conf, but isn't documented
# there.  Normally no debugs are in this script, but it can
# help development and customer debug
typeset -i debug=0

# root filesytem usage (df --output=pcent /)
typeset -i diskalarm=90
typeset -i diskcrit=95
# system memory usage (memory used / total memory) * 100
typeset -i memalarm=90
typeset -i memcrit=95
# total cpu utilization over all cores; as a percent
typeset -i cpualarm=80
typeset -i cpucrit=95
# (5 minute load avg / number of cores) * 100
typeset -i loadalarm=95
typeset -i loadcrit=125

[ -d $svcdir ] || mkdir -p $svcdir

# read limits, if file is present
[ -f /etc/cumulus/${svc}.conf ] && . /etc/cumulus/${svc}.conf

# after sysmonitor and before status, so customer errors in
# config file don't affect correctness

typeset -i prevdiskalarm=0
typeset -i prevdiskcrit=0
typeset -i prevmemalarm=0
typeset -i prevmemcrit=0
typeset -i prevcpualarm=0
typeset -i prevcpucrit=0
typeset -i prevloadalarm=0
typeset -i prevloadcrit=0

typeset -i ncores=0

# for system cpu usage
typeset -i prevuptime=0
typeset -i previdletime=0


runstatus=${svcdir}/status 

# read previous status
[ -f ${runstatus} ] && . ${runstatus}

function log()
{
    local pri=$1; shift
    logger -t ${svc} -p $pri -- $@
}

function debug()
{
    [ ${debug} -gt 0 ] && logger -s -t ${svc} -p debug -- $@
}

function savestate()
{
    local f vars

    vars="prevdiskalarm prevdiskcrit prevmemalarm prevmemcrit prevcpualarm prevcpucrit
     prevloadalarm prevloadcrit ncores prevuptime previdletime"
    for f in ${vars}; do
       echo ${f}'='${!f}
    done > ${runstatus}.tmp && mv ${runstatus}.tmp ${runstatus}
}

# check (only) root filesystem for being overly full
# don't complain repeatedly if there is not a significant status
# change
function chkrootfs()
{
    local -i rootpct

    rootpct=$(df --output=pcent / | sed -n '$s/%//p')
    if [ ${rootpct} -ge ${diskcrit} ]; then 
	if [ ${rootpct} -gt ${prevdiskcrit} ]; then 
	    log crit / filesystem critically full: ${rootpct}'%' in use
	fi
	prevdiskcrit=${rootpct} prevdiskalarm=
    elif [ ${rootpct} -ge ${diskalarm} ]; then
	if [ ${rootpct} -gt ${prevdiskalarm} ]; then 
	    log alert / filesystem nearly full: ${rootpct}'%' in use
	fi
	prevdiskalarm=${rootpct} prevdiskcrit=
    else
       if [ ${prevdiskcrit} -gt 0 ]; then
	    log notice / filesystem no longer critically full: ${rootpct}'%' in use
       elif [ ${prevdiskalarm} -gt 0 ]; then
	    log notice / filesystem no longer nearly full: ${rootpct}'%' in use
       fi
       prevdiskalarm=  prevdiskcrit=
    fi
}

# system memory usage (memory used / total memory) * 100
function chkmem()
{
   local -i memtot=0 memfree=0 val
   local field unit
   while read field val unit; do
	case $field in
	MemTotal*) memtot=$val ;;
	MemAvailable*)  memfree=$val ;;
	esac
	[ $memtot -gt 0 -a $memfree -gt 0 ] && break
   done < /proc/meminfo

   ((pctused = (($memtot - $memfree)*100) / $memtot ))

    if [ ${pctused} -ge ${memcrit} ]; then 
	if [ ${pctused} -gt ${prevmemcrit} ]; then 
	    log crit Critically low free memory: ${pctused}'%' in use
	fi
	prevmemcrit=${pctused} prevmemalarm=
    elif [ ${pctused} -ge ${memalarm} ]; then
	if [ ${pctused} -gt ${prevmemalarm} ]; then 
	    log alert Low free memory: ${pctused}'%' in use
	fi
	prevmemalarm=${pctused} prevmemcrit=
    else
       if [ ${prevmemcrit} -gt 0 ]; then
	    log notice Free memory no longer critically low: ${pctused}'%' in use
       elif [ ${prevmemalarm} -gt 0 ]; then
	    log notice Free memory no longer low: ${pctused}'%' in use
       fi
       prevmemalarm=  prevmemcrit=
    fi
}

# (5 minute load avg / number of cores) * 100
function chkload()
{
   local -i min fiv fteen
   local mind fivd fteend dispavgd updispavgd # fractional part not declared
	    # numeric because it can have a leading 0, and don't want read
        # to get an error
   local -i dispavg updispavg
   local rest avg upavg

   # we need to determine number of cores if we don't already have it
   [ $ncores -eq 0 ] && ncores=$(grep -c '^core id' /proc/cpuinfo)
   [ $ncores -eq 0 ] && ncores=1 # have to assume something...

   IFS=" ." read min mind fiv fivd fteen fteend rest < /proc/loadavg
   # force fivd to be treated as decimal below
   (( avg5 = ((100*$fiv) + 10#$fivd) / $ncores ))
   (( upavg5 = $ncores * $avg5 )) # loadavg style, for clarity
   (( dispavg = $avg5 / 100 ))
   (( dispavgd = $avg5 % 100 ))
   (( updispavg = $upavg5 / 100 ))
   (( updispavgd = $upavg5 % 100 ))
   [ $dispavgd -lt 10 ] && dispavgd=0${dispavgd}
   [ $updispavgd -lt 10 ] && updispavgd=0${dispavgd}
   avg=${dispavg}.${dispavgd}
   upavg=${updispavg}.${updispavgd}
  
    if [ ${avg5} -ge ${loadcrit} ]; then 
	if [ ${avg5} -gt ${prevloadcrit} ]; then 
	    log crit Critically high load average: ${avg} "(${upavg})"
	fi
	prevloadcrit=${avg5} prevloadalarm=
    elif [ ${avg5} -ge ${loadalarm} ]; then
	if [ ${avg5} -gt ${prevloadalarm} ]; then 
	    log alert High load average: ${avg} "(${upavg})"
	fi
	prevloadalarm=${avg5} prevloadcrit=
    else
       if [ ${prevloadcrit} -gt 0 ]; then
	    log notice Load Average no longer critically high: ${avg} "(${upavg})"
       elif [ ${prevloadalarm} -gt 0 ]; then
	    log notice Load Average no longer high: ${avg} "(${upavg})"
       fi
       prevloadalarm=  prevloadcrit=
    fi
}

# check total cpu utilization over all cores
function chkcpu()
{
   local -i up idle val thisup thisidle
   local up_d idle_d # fractional part not declared numeric
	# because it can have a leading 0, and don't want readd
        # to get an error
   local rest

   # we need to determine number of cores if we don't already have it
   [ $ncores -eq 0 ] && ncores=$(grep -c '^core id' /proc/cpuinfo)
   [ $ncores -eq 0 ] && ncores=1 # have to assume something...

   IFS=" ." read up up_d idle idle_d rest < /proc/uptime

   (( up = (${up} * 100) + 10#${up_d} ))
   (( idle = (${idle} * 100) + 10#${idle_d} ))

   # If this is our first pass, we can't calculate cpu use, just init
   if [ ${prevuptime} -eq 0 -o ${previdletime} -eq 0 ] ; then
	prevuptime=${up}
	previdletime=${idle}
        return
   fi
   thisup=${up}
   thisidle=${idle}
   (( up -= ${prevuptime} ))
   (( idle -= ${previdletime} ))
   (( idle /= ${ncores} ))
   (( cpu = (100 * (${up} - ${idle})) / ${up} ))
    prevuptime=${thisup}
    previdletime=${thisidle}

    if [ ${cpu} -ge ${cpucrit} ]; then 
	if [ ${cpu} -gt ${prevcpucrit} ]; then 
	    log crit Critically high CPU use: ${cpu}'%'
	fi
	prevcpucrit=${cpu} prevcpualarm=
    elif [ ${cpu} -ge ${cpualarm} ]; then
	if [ ${cpu} -gt ${prevcpualarm} ]; then 
	    log alert High CPU use: ${cpu}'%'
	fi
	prevcpualarm=${cpu} prevcpucrit=
    else
       if [ ${prevcpucrit} -gt 0 ]; then
	    log notice CPU use no longer critically high: ${cpu}'%'
       elif [ ${prevcpualarm} -gt 0 ]; then
	    log notice CPU use no longer high: ${cpu}'%'
       fi
       prevcpualarm=  prevcpucrit=
    fi
}

# main execution starts here

debug Limits are diskcrit=${diskcrit}, diskalarm=${diskalarm}, \
  memalarm=${memalarm}, memcrit=${memcrit}, cpualarm=${cpualarm}, \
  cpucrit=${cpucrit}, loadalarm=${loadalarm}, loadcrit=${loadcrit}.

debug Previous event values are prevdiskalarm=${prevdiskalarm}, \
    prevdiskcrit=${prevdiskcrit}, prevmemalarm=${prevmemalarm}, \
    prevmemcrit=${prevmemcrit}, prevcpualarm=${prevcpualarm}, \
    prevcpucrit=${prevcpucrit} prevloadalarm=${prevloadalarm}, \
    prevloadcrit=${prevloadcrit} prevuptime=${prevuptime},
    previdletime=${previdletime}

chkmem
chkrootfs
chkload
chkcpu

savestate

exit 0 # Don't want systemd to disable if we "fail"
