#!/bin/ksh
# File	: chk_snap-mirror-vault_lag.ksh
# By	: Maarten.deBoer@AtosOrigin.com, (090804) 131210
# Subject	: Check SnapMirror & Vault lags
#(0.2): Made .sh
#(0.3): Check sizes on both filers. Can differ due to usage of autosizing
#(0.4): Check also snapmirror.conf & MAILFILE added
#(0.5): Mod. get SRCFILER & snapmirror.conf from vfilers
#(0.6): Log is now 25 (iso 24) hr. USAGE & -f added
#(0.7): No size check of Broken-off SM's
#(0.8): Added --mailto
#(0.1): Copied from chk_snapmirrors.sh
#(0.2): Added --sm & --sv
#(0.3): Added vfilers (vfiler0 .. rest), LOGGER + CI
#(0.4): Added option snapmirror.enable=off
#(0.5): Added; get snapmirror.conf info at ERROR
#(0.6): Added SV-error status & SUBJECT, ERROR at SM
#(0.7): Mod. empty ERR-file, mod; grep to egrep, Added VFILER at SV-check, Add "cat ${ERR}|sort -u >> ${TMPMAIL}"
#
# Wentto KSH (in stead of SH) because of the calculations
# Check if SnapMirror lags are to long (> ${LAG}).
#
### Check also if volume size of both SnapMirrored volumes became not equal, 
### due to autosizing of one (source) of them # And mail the output

PGM="`basename $0|cut -d\. -f1`" 
VER="0.7"
TMP="/tmp/${PGM}.$$"
ERR="/tmp/${PGM}.$$.err"
LOG="${HOME}/log/${PGM}.log"
HOSTNAME="`hostname|cut -d\. -f1`"
FILERS="${HOME}/etc/filers"
MAIL=""
MAILTO="maarten.deboer@atos.net"
FILTER="[?]*"
MAXLOGSIZE=1024   # In K's
SSH="/usr/bin/ssh -n"
MAXMIRRORLAG="1500"  # In Minutes =25hr
MAXVAULTLAG="25"  # In Hours
TMPMAIL="${TMP}.mail"
VERBOSE=""
SMONLY="1"
SVONLY="1"
CI=${HOSTNAME}
LOGGER=""
MAXSMDISLAG=240
NAGIOSSEV="WARNING"
SUBJECT="@${HOSTNAME}: Check "


USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options:"
  echo "    -e             : Etc/filers (${FILERS})"
  echo "    -f             : Filter filername (${FILTER})"
  echo "    -h | --help    : this help"
  echo "    --logger       : send to nagios via Logger (message-file)"
  echo "    -m | --mail    : do send mail"
  echo "    --maxmirrorlag : maximum snapMirror Lag in minutes (${MAXMIRRORLAG})"
  echo "    --maxvaultlag  : maximum snapVault Lag in hours (${MAXVAULTLAG})"
  echo "    --sm           : SnapMirror only"
  echo "    --sv           : SnapVault only"
  echo "    -v             : Verbose mode"
  echo "    -V             : Version"
  echo "    -x             : set -x"
  echo "    --mailto       : change MAILTO address & do send mail (${MAILTO})"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -e) FILERS=${2}; shift ;;
    -f) FILTER=${2}; shift ;;
    -h | --help) USAGE; exit 1 ;;
    --logger) LOGGER=1 ;;
    -m | --mail) MAIL=1 ;;
    --maxmirrorlag) MAXMIRRORLAG=${2}; shift ;;
    --maxvaultlag) MAXVAULTLAG=${2}; shift ;;
    --sm) SVONLY="" ;;
    --sv) SMONLY="" ;;
    -v) VERBOSE=1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    --mailto) MAILTO=$2; MAIL=1; shift ;;
    -x)  set -x ;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

echo "`date` ${PGM} v${VER} started"|tee -a ${LOG}|tee -a ${TMPMAIL}
echo ""|tee -a ${TMPMAIL}
echo "FILERS=${FILERS}"|tee -a ${TMPMAIL}
echo "FILTER=${FILTER}"|tee -a ${TMPMAIL}
echo "HOSTNAME=${HOSTNAME}"|tee -a ${TMPMAIL}
echo "MAXMIRRORLAG=${MAXMIRRORLAG}"|tee -a ${TMPMAIL}
echo "MAXVAULTLAG=${MAXVAULTLAG}"|tee -a ${TMPMAIL}
echo "MAXSMDISLAG=${MAXSMDISLAG}"|tee -a ${TMPMAIL}
echo "LOGGER=${LOGGER}"|tee -a ${TMPMAIL}
echo "LOG=${LOG}"|tee -a ${TMPMAIL}
echo "MAIL=${MAIL}"|tee -a ${TMPMAIL}
echo "MAILTO=${MAILTO}"|tee -a ${TMPMAIL}
echo "SMONLY=${SMONLY}"|tee -a ${TMPMAIL}
echo "SVONLY=${SVONLY}"|tee -a ${TMPMAIL}
echo "VERBOSE=${VERBOSE}"|tee -a ${TMPMAIL}

echo "  MAXMIRRORLAG=${MAXMIRRORLAG}min., MAXVAULTLAG=${MAXVAULTLAG}hrs., MAXSMDISLAG=${MAXSMDISLAG}" >> ${LOG}


#  LOG rotating
touch ${LOG}
# Check & move LOG-file if longer then max.
LOGSIZE=`du -ka $LOG | cut -f1`
if [ $LOGSIZE -ge $MAXLOGSIZE ]; then
  mv $LOG $LOG.old
  touch $LOG
fi

if [ ! -f ${FILERS} ]; then
  echo "Etc-file ${FILERS} NOT found. Exiting ..."|tee -a ${LOG}
  exit 3
fi  # ${FILERS}

touch ${TMP} ${ERR}
for FILER in `cat ${FILERS}|grep -v \#|awk -F\; '{print$1}'|grep ${FILTER}`
do
# Empty ${TMP} & ${ERR} (per filer, because of CIs)
  cp /dev/null ${ERR}
  OPTIONS="`${SSH} ${FILER} options snapmirror.enable|awk '{print $2}'`"
  if [ "${OPTIONS}" = "on" ]; then
    SMENABLED=1
  else
    SMENABLED=""
  fi
  OPTIONS="`${SSH} ${FILER} options snapvault.enable|awk '{print $2}'`"
  if [ "${OPTIONS}" = "on" ]; then
    SVENABLED=1
  else
    SVENABLED=""
  fi

  if [ ${SMONLY} ]; then
# SnapMirror
# Empty ${TMP} 
    cp /dev/null ${TMP}
    echo "" >> ${ERR}
    echo "`date` SnapMirror-lag check of (destination) ${FILER} (Max.Mirror-Lag=${MAXMIRRORLAG}min.) :" >> ${ERR}

    ${SSH} ${FILER} vfiler status|grep running|awk '{print $1}'|while read VFILER
    do
      echo "${FILER}/${VFILER} SnapMirror ... "
      ${SSH} ${FILER} vfiler run ${VFILER} snapmirror status|grep ':'|grep -v Source|grep -v Broken-off|grep -v Uninitialized|grep -v Unknown >> ${TMP}
    done  # VFILER

    echo "   analyzing ..."
    cat ${TMP}| while read LINE
    do
      if [ ${VERBOSE} ]; then
        echo "${LINE}"
      fi
      SRCFILER="`echo ${LINE} |awk '{print $1}'|cut -d\: -f1|cut -d\- -f1`"
      SRCVOL="`echo ${LINE} |awk '{print $1}'|cut -d\: -f2`"
      DESTFILER="`echo ${LINE} |awk '{print $2}'|cut -d\: -f1|cut -d\- -f1`"
      DESTVOL="`echo ${LINE} |awk '{print $2}'|cut -d\: -f2`"
      CURLAGHR="`echo ${LINE} |awk '{print $4}'|cut -d\: -f1`"
      CURLAGMIN="`echo ${LINE} |awk '{print $4}'|cut -d\: -f2`"
      let "CURLAGMINUTES = (${CURLAGHR} * 60) + ${CURLAGMIN}"

#    echo "SRCFILER=${SRCFILER} SRCVOL=${SRCVOL} DESTFILER=${DESTFILER} DESTVOL=${DESTVOL} CURLAG=${CURLAGHR}:${CURLAGMIN} -> ${CURLAGMINUTES}"

      if [ ${CURLAGMINUTES} -gt ${MAXMIRRORLAG} ]; then
# Get ERR
        ERROR=`${SSH} ${DESTFILER} snapmirror status -l ${DESTVOL}|grep 'Current Transfer Error:'|cut -d: -f2`
        NAGIOSSTR="CI=${DESTFILER} SnapMirror-Lag ${SRCFILER}:${SRCVOL} to ${DESTFILER}:${DESTVOL} is to HIGH(${MAXMIRRORLAG}min.) Curr=${CURLAGMINUTES}min. (${CURLAGHR}:${CURLAGMIN}). ERROR=${ERROR}"

# If SM is enabled, then Alert.
        if [ ${SMENABLED} ]; then
          echo "ERROR: ${NAGIOSSTR}"|tee -a ${ERR}|tee -a ${LOG}
          echo -n "  SM.conf: "|tee -a ${ERR}|tee -a ${LOG}
          ${SSH} ${DESTFILER} rdfile /etc/snapmirror.conf|grep ${DESTVOL}|tee -a ${ERR}|tee -a ${LOG}
          if [ ${LOGGER} ]; then
            /usr/bin/logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} "${NAGIOSSTR}"
            echo "`date`: logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} ${NAGIOSSTR}"|tee -a ${LOG}
          fi  # LOGGER
        else
# Else check is LAG > MAXSMDISLAG
# Is max lag, when SM is disabled.
          if [ ${CURLAGMINUTES} -gt ${MAXSMDISLAG} ]; then
            echo "ERROR(SMdis): ${NAGIOSSTR}"|tee -a ${ERR}|tee -a ${LOG}
            if [ ${LOGGER} ]; then
              /usr/bin/logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} "${NAGIOSSTR}"
              echo "`date`: logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} ${NAGIOSSTR}"|tee -a ${LOG}
            fi  # LOGGER
          fi #  ${CURLAGMINUTES} -gt ${MAXMIRRORLAG}
        fi  # SMENABLED

      fi  # if .. ${MAXMIRRORLAG}
    done  # while read LINE .. SnapMirror
  fi  # SMONLY

  if [ ${SVONLY} ]; then
# SnapVault
# Empty ${TMP} 
    cp /dev/null ${TMP}
    echo "" >> ${ERR}
    echo "`date` SnapVault-lag check of (destination) ${FILER} (Max.Vault-Lag=${MAXVAULTLAG}hrs.) :" >> ${ERR}

    ${SSH} ${FILER} vfiler status|grep running|awk '{print $1}'|while read VFILER
    do
      echo "${FILER}/${VFILER} SnapVault ... "
      ${SSH} ${FILER} vfiler run ${VFILER} snapvault status|grep ':'|egrep -v 'Source|Broken-off|Uninitialized|Unknown' >> ${TMP}
    done  # VFILER

    echo "   analyzing ..."
    cat ${TMP}|while read LINE
    do
      if [ ${VERBOSE} ]; then
        echo "${LINE}"
      fi
      SRCFILER="`echo ${LINE} |awk '{print $1}'|cut -d\: -f1|cut -d\- -f1`"
      SRCVOL="`echo ${LINE} |awk '{print $1}'|cut -d\: -f2`"
      DESTFILER="`echo ${LINE} |awk '{print $2}'|cut -d\: -f1|cut -d\- -f1`"
      DESTVOL="`echo ${LINE} |awk '{print $2}'|cut -d\: -f2`"
      CURLAGHR="`echo ${LINE} |awk '{print $4}'|cut -d\: -f1`"
      CURLAGMIN="`echo ${LINE} |awk '{print $4}'|cut -d\: -f2`"
      let "CURLAGHOURS = (${CURLAGHR})"

#    echo "SRCFILER=${SRCFILER} SRCVOL=${SRCVOL} DESTFILER=${DESTFILER} DESTVOL=${DESTVOL} CURLAG=${CURLAGHR}:${CURLAGMIN} -> ${CURLAGHOURS}"

      if [ ${CURLAGHOURS} -gt ${MAXVAULTLAG} ]; then
# Get ERR
        ERROR=`${SSH} ${DESTFILER} snapvault status -l ${DESTVOL}|grep 'Current Transfer Error:'|cut -d: -f2`
        NAGIOSSTR="CI=${DESTFILER} SnapVault-Lag ${SRCFILER}:${SRCVOL} to ${DESTFILER}:${DESTVOL} is to HIGH(${MAXVAULTLAG}hr.) Curr=${CURLAGHOURS}hrs. (${CURLAGHR}:${CURLAGMIN}). ERROR=${ERROR}"
        echo "ERROR: ${NAGIOSSTR}"|tee -a ${ERR}|tee -a ${LOG}
        if [ ${LOGGER} ]; then
          /usr/bin/logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} "${NAGIOSSTR}"
          echo "`date`: logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} ${NAGIOSSTR}"|tee -a ${LOG}
        fi  # LOGGER
      fi  # if .. ${MAXVAULTLAG}

    done  # while read LINE .. SnapVault
  fi  # SVONLY

  cat ${ERR}|sort -u >> ${TMPMAIL}
# Empty ERR-file
  cp /dev/null ${ERR}
done  # for FILER

if [ ${SMONLY} ]; then
 SUBJECT="${SUBJECT} SnapMirror "
fi
if [ ${SVONLY} ]; then
 SUBJECT="${SUBJECT} SnapVault "
fi
SUBJECT="${SUBJECT} lags [${PGM} v${VER}]"

if [ ${MAIL} ]; then
  cat ${TMPMAIL} | mailx -s "${SUBJECT}" "${MAILTO}"
  echo "Mail (${SUBJECT}) had been send to ${MAILTO} ..." | tee -a ${LOG}
fi

echo "`date` ${PGM} v${VER} finished"| tee -a ${LOG}

# Cleanup
rm ${TMP} ${ERR} ${TMPMAIL} 
exit 0