scripts/innwatch.in

   1 #! /bin/sh
   2 # fixscript will replace this line with code to load innshellvars
   3
   4 ##  $Revision: 2677 $
   5 ##  Watch the state of the system relative to the news subsystem.
   6 ##  As controlled by the control file, when space or inodes are almost
   7 ##  exhausted innd is throttled, or paused, similarly if the load is
   8 ##  too high - when conditions revert to normal, innd is restarted.
   9 ##  No logging is done here, watch for syslog reports from innd.
  10 ##  Written by Mike Cooper <mcooper@usc.edu>.
  11 ##  Extensively modified by <kre@munnari.oz.au>.
  12 ##  Watch a log file and send mail when it gets new output by
  13 ##      Steve Groom <stevo@elroy.Jpl.Nasa.Gov>
  14 ##  Steve's extensions merged in innwatch by
  15 ##      <Christophe.Wolfhugel@grasp.insa-lyon.fr>
  16
  17 PROGNAME=innwatch
  18 LOCK=${LOCKS}/LOCK.${PROGNAME}
  19 DAILY=${LOCKS}/LOCK.news.daily
  20 ##  Where to put the timestamp file (directory and filename).
  21 TIMESTAMP=${LOCKS}/${PROGNAME}.time
  22
  23 ##  Logfile to watch. Comment out if no logwatch.
  24 LOGFILE=${MOST_LOGS}/news.crit
  25
  26 ## Default value in case there is no definition in inn.conf
  27 : ${INNWATCHPAUSELOAD:=1500}
  28 : ${INNWATCHHILOAD:=2000}
  29 : ${INNWATCHLOLOAD:=1000}
  30 : ${INNWATCHSPOOLSPACE:=8000}
  31 : ${INNWATCHBATCHSPACE:=800}
  32 : ${INNWATCHLIBSPACE:=25000}
  33 : ${INNWATCHSPOOLNODES:=200}
  34 : ${INNWATCHSLEEPTIME:=600}
  35
  36 ##  Parse JCL.
  37 while [ $# -gt 0 ] ; do
  38     case X"$1" in
  39     X-f)
  40         FILE=$2
  41         shift
  42         ;;
  43     X-f*)
  44         FILE=`expr "$1" : '-s\(.*\)'`
  45         ;;
  46     X-l)
  47         LOGFILE=$2
  48         shift
  49         ;;
  50     X-l*)
  51         LOGFILE=`expr "$1" : '-s\(.*\)'`
  52         ;;
  53     X-t)
  54         INNWATCHSLEEPTIME=$2
  55         shift
  56         ;;
  57     X-t*)
  58         INNWATCHSLEEPTIME=`expr "$1" : '-t\(.*\)'`
  59         ;;
  60     X--)
  61         shift
  62         break
  63         ;;
  64     X-*)
  65         echo "${PROGNAME}:  Unknown flag $1" 1>&2
  66         exit 1
  67         ;;
  68     *)
  69         break
  70         ;;
  71     esac
  72     shift
  73 done
  74
  75 ##  Process arguments.
  76 if [ $# -ne 0 ] ; then
  77     echo "Usage:  ${PROGNAME} [flags]" 1>&2
  78     exit 1
  79 fi
  80
  81 trap '' 2
  82
  83 ##  Anyone else there?
  84 shlock -p $$ -f ${LOCK} || {
  85     echo "${PROGNAME}: [$$] locked by [`cat ${LOCK}`]"
  86     exit 0
  87 }
  88
  89 trap 'rm -f ${LOCK} ${WATCHPID} ; exit 1' 1 3 15
  90 echo "$$" > ${WATCHPID}
  91
  92 ##  The reason why we turned INND off, and its, and our current state.
  93 REASON=''
  94 INND=''
  95 STATE=''
  96
  97 trap '(
  98         echo "${PROGNAME} waiting for INND to start (pid: $$)"
  99         date
 100     ) >${INNWSTATUS}' 2
 101
 102 ##  We need to remember the process ID of innd, in case one exits
 103 ##  But we need to wait for innd to start before we can do that
 104 while PID=`cat ${SERVERPID} 2>/dev/null`; test -z "${PID}"; do
 105     sleep ${INNWATCHSLEEPTIME}
 106 done
 107
 108 trap '(
 109         if [ -z "${STATE}" ]; then
 110             echo "${PROGNAME} state RUN interval ${INNWATCHSLEEPTIME} pid $$"
 111         else
 112             echo "${PROGNAME} state ${STATE} interval ${INNWATCHSLEEPTIME} pid $$"
 113         fi
 114         if [ -z "${INND}" ]; then
 115             X=GO
 116         else
 117             X="${INND}"
 118         fi
 119         test -n "${REASON}" && X="${X}: ${REASON}"
 120         echo "INND state ${X}"
 121         date
 122     ) >${INNWSTATUS}' 2
 123
 124 cd ${SPOOL}
 125
 126 NEXTSLEEP=1
 127 HASEXITED=false
 128
 129 while { sleep ${NEXTSLEEP} & wait; } ; : ; do
 130     NEXTSLEEP=${INNWATCHSLEEPTIME}
 131
 132     ##  If news.daily is running, idle:  we don't want to change the
 133     ##  status of anything while news.daily may be depending on what we
 134     ##  have done.
 135     test -f "${DAILY}" && continue
 136
 137     ## Check to see if INND is running.
 138     ## Notify NEWSMASTER if it has stopped or just restarted.
 139     if ctlinnd -s -t 120 mode 2>/dev/null ; then
 140         ${HASEXITED} && {
 141             HASEXITED=false
 142             ${MAILCMD} -s "INND is now running" ${NEWSMASTER} </dev/null
 143         }
 144     else
 145         ${HASEXITED} || {
 146             HASEXITED=true
 147             ${MAILCMD} -s "INND is NOT running" ${NEWSMASTER} </dev/null
 148         }
 149         continue
 150     fi
 151
 152     ##  If innd has exited & restarted, put the new one into the
 153     ##  same state the old one was in
 154
 155     nPID=`cat ${SERVERPID} 2>/dev/null`
 156     test -n "${nPID}" -a "${PID}" -ne "${nPID}" && {
 157         test -n "${INND}" -a "${INND}" != go && ctlinnd -s "${INND}" "${REASON}"
 158         PID="${nPID}"
 159     }
 160
 161     VALUE=0
 162     PREVEXP=''
 163
 164     exec 3<&0
 165     exec 0<${CTLWATCH}
 166
 167     LINE=0
 168     while read line ; do
 169         LINE=`expr ${LINE} + 1`
 170         test -z "$line" && continue
 171
 172         ##  The first character on the line is the field delimiter,
 173         ##  except '#' which marks the line as a comment
 174         delim=`expr "${line}" : '\(.\).*'`
 175         test "X${delim}" = 'X#' && continue
 176
 177         ##  Parse the line into seven fields, and assign them to local vars.
 178         ##  You're welcome to work out what's going on with quoting in
 179         ##  the next few lines if you feel inclined.
 180         eval `trap '' 2; echo "${line}" \
 181                 | ${SED} -e "s/'/'\"'\"'/g" \
 182                     -e "s/[     ]*\\\\${delim}[         ]*/\\\\${delim}/g" \
 183                 | ${AWK} -F"${delim}" '{ print  "LAB='"'"'" $2 "'"'"'", \
 184                                                 "CND='"'"'" $3 "'"'"'", \
 185                                                 "EXP='"'"'" $4 "'"'"'", \
 186                                                 "TST='"'"'" $5 "'"'"'", \
 187                                                 "LIM=" $6, \
 188                                                 "CMD='"'"'" $7 "'"'"'", \
 189                                                 "CMT='"'"'" $8 "'"'"'" }'`
 190
 191         ##  If there's no label, the label is the line number.
 192         test -z "${LAB}" && LAB=${LINE}
 193
 194         ##  Should we act on this line?  We will if one (or more) of the
 195         ##  specified conditions is satisfied.
 196         for X in a b; do        # meaningless trash because we have no goto
 197             if [ -z "${CND}" ]; then
 198                 X=-
 199             else
 200                 X="${CND}"
 201             fi
 202             set -$- X ${X}; shift
 203             for cnd
 204             do
 205                 case "${cnd}" in
 206                 -)
 207                     test -n "${STATE}" -a "X${STATE}" != "X${LAB}" && continue
 208                     ;;
 209                 +)
 210                     test -n "${STATE}" && continue
 211                     ;;
 212                 '*')
 213                     ;;
 214                 -*)
 215                     test "X-${STATE}" = "X${cnd}" && continue
 216                     ;;
 217                 *)
 218                     test "X${STATE}" != "X${cnd}" && continue;
 219                     ;;
 220                 esac
 221                 break 2;        # OK, continue with this line
 222             done
 223             continue 2;         # No, skip it.
 224         done
 225
 226         ##  Evaluate the expression, if there is one, and if that works.
 227         if [ -z "${EXP}" -o "${EXP}" = "${PREVEXP}" ] \
 228          || { PREVEXP="${EXP}"; VALUE=`trap '' 2;eval "${EXP}"`; }; then
 229             ##  If innd is running, and test "succeeds", stop it.
 230             case "${CMD}" in
 231             throttle|pause)
 232                 OK=n
 233                 ;;
 234             *)
 235                 OK=y
 236                 ;;
 237             esac
 238
 239             if [ \( -z "${STATE}" -o "${STATE}" != "${LAB}" -o "${OK}" = y \) \
 240                     -a "${VALUE}" "-${TST}" "${LIM}" ] ; then
 241                 R="${CMT} [${PROGNAME}:${LAB}] ${VALUE} ${TST} ${LIM}"
 242                 O=
 243                 case "${CMD}" in
 244                 throttle)
 245                     case "${STATE}" in
 246                     ''|go)
 247                         REASON="${R}"
 248                         ;;
 249                     *)
 250                         ;;
 251                     esac
 252                     O="${LAB}"
 253                     ARG="${REASON}"
 254                     ;;
 255                 pause)
 256                     O="${LAB}"
 257                     REASON="${R}"
 258                     ARG="${REASON}"
 259                     ;;
 260                 shutdown)
 261                     ARG="${R}"
 262                     ;;
 263                 flush)
 264                     ARG=''
 265                     O="${STATE}"
 266                     ARG="${REASON}"
 267                     ;;
 268                 go)
 269                     ARG="${REASON}"
 270                     NEXTSLEEP=1
 271                     REASON=''
 272                     ;;
 273                 exit)
 274                     exit 0
 275                     ;;
 276                 *)
 277                     break
 278                     ;;
 279                 esac
 280
 281                 ctlinnd -s "${CMD}" "${ARG}" && STATE="${O}" && INND="${CMD}"
 282                 break
 283
 284             ##  Otherwise, if innd is not running, and reverse test succeeds
 285             ##  restart it.
 286             elif [ "${STATE}" = "${LAB}" -a \
 287                     \( "${CMD}" = "throttle" -o "${CMD}" = pause \) -a \
 288                     ! "${VALUE}" "-${TST}" "${LIM}" ] ; then
 289                 ctlinnd -s go "${REASON}"
 290                 STATE=''
 291                 REASON=''
 292                 INND=''
 293
 294                 ##  If we have started innd, run all tests again quickly in
 295                 ##  case there is some other condition that should stop it.
 296                 NEXTSLEEP=1
 297                 break
 298             fi
 299         fi
 300
 301     done
 302
 303     exec 0<&3
 304     exec 3<&-
 305
 306     if [ -n "${LOGFILE}" -a -f "${LOGFILE}" ]; then
 307         if [ ! -f ${TIMESTAMP} ]; then
 308             DOIT=${LOGFILE}
 309         else
 310             # use ls to print most recently modified file first.
 311             # If that's ${LOGFILE}, it's changed since the last pass.
 312             DOIT="`ls -t ${TIMESTAMP} ${LOGFILE} | ${SED} -e 1q | grep ${LOGFILE}`"
 313         fi
 314
 315         # If the file has been modified more recently than the timestamp,
 316         # and the file has length greater than 0, send the warning.
 317         if [ -n "${DOIT}" -a -s ${LOGFILE} ]; then
 318             date >${TIMESTAMP}
 319             (
 320                 ls -l ${LOGFILE}
 321                 echo "-----"
 322                 ctlinnd -t120 mode
 323                 echo "-----"
 324                 cat ${LOGFILE}
 325             ) 2>&1 \
 326             | sed -e 's/^~/~~/' \
 327             | ${MAILCMD} -s "${PROGNAME} warning: messages in ${LOGFILE}" \
 328                 ${NEWSMASTER}
 329         fi
 330     fi
 331
 332 done
 333
 334 rm -f ${LOCK}