#!/bin/sh
# File	: chk_filer_daily.sh
# By	: Maarten.deBoer@AtosOrigin.com, 091208, 091214
# Subject	: Check daily some items which are not covered by DFM
# Checks
#1) If RML/BCM/Console-connectie works
#2) filername = hostname. So its check if IP-adres has the correct filer-name
#3) Is ROOTVOL ($ETC)works. Needed for configuration actions
#4) Is autosupport=on ?
#5) Is filer-time = server-time ? Agv P-UK:8458681
# (0.2)	: At FSOD; Added ping to RLM/BMC/console (<filername>-rlm)
#(0.3)  : If no -rlm configured
#(0.4)  : Added; check autosupport, Check ${SSH} & S{PING}
#(0.5)	: changed name from chk_connects.sh => chk_filer_daily.sh. Added: time(-skew) caused by P-UK:8458681 
#(0.6)	: Added SMS-time-ERR's & --testsms, MAXLOGSIZE
#(0.7)  : Added HOSTDATE & FILERDATE check
#(0.8)  : Check uppercase hostname
#(0.9)	: PATH added, because of PING
#(0.10)	: Added mail2USD & USAGE & LOGGER(to USD), date made consistent
# 	MSGSTR added. Used for SMS & LOGGER-msgs. Logger & SMS can only be used with short msgs. Long messages, should go via (IMI)USD(MAIL)
#	if [ "`who am i | awk '{print $1}'`" != "nl19471" ];
#(0.11)	: Mod. for RHEL6 (mailx -a). No direct SMS/GIN (not allowed by Atos); date
#(0.12)	: Mod.; FILTER added. Logger /CI
#set -x

PGM="`basename $0|cut -d\. -f1`"
VERSION="0.12"
HOSTNAME="`hostname`"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
ERR="/tmp/${PGM}.$$.err"
FILERS="${HOME}/etc/filers"
MAIL=""
MAILTO="maarten.deboer@atos.net"
MAXLOGSIZE=1024   # In K's
USD=""
USDMAIL="prod-imi@gis.nl.ao-srv.com"
USDUSER="nl19471" 
USDMSG="${TMP}.usd"
USDREPLYTO="maarten.deboer@atos.net"
LOGGER=""
LOGGERMSG=""
MSGSTR=""
MESSAGESFILE="/var/log/messages"
SSH="/usr/bin/ssh -n"
FILTER="[0123456789]*"


# if not me, then change some user related info
MAILTO="fsod@atos.net"
USDUSER="nldsm01" 
USDREPLYTO="fsod@atos.net"

USAGE()
{
  echo "Usage: ${PGM} [<option(s)>] "
  echo "  Version: ${VERSION}"
  echo "  options         :"
  echo "    -f            : filter filername (${FILTER})"
  echo "    -h | --help   : this help"
  echo "    -m | --mail   : do send mail (to ${MAILTO})"
  echo "    -l | --logger : send (short)msg (to USD) via Logger (, Nagios & ANGate)"
  echo "    -u | --usd    : send (long) msg (to USD) via IMI"
  echo "    -V            : Version"
  echo "    -x            : set -x"
}

# Check options
while [ ${#} -gt 0 ]
  do
  case ${1} in
    -f) FILTER=$2; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -l | --logger) LOGGER=1;;
    -m | --mail) MAIL=1;;
    -u | --usd) USD=1;;
    -V) echo "${PGM}: v${VERSION}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

touch ${LOG} ${TMP} ${ERR} ${USDMSG}

SSH="`type ssh | awk '{print $3}'`"
PING="`type ping | awk '{print $3}'`"
if [ "${SSH}" = "found" ] || [ "${PING}" = "found" ] ; then
  echo "No SSH (${SSH}) or PING (${PING}) found. Exiting ..."|tee -a $LOG
  exit 2
fi

# Check & move LOG-file if longer then max.
LOGSIZE=`du -ka ${LOG} | cut -f1`
if [ ${LOGSIZE} -ge ${MAXLOGSIZE} ]; then
  mv ${LOG} ${LOG}.old
  touch ${LOG}
fi

echo "`date` ${PGM} (v${VERSION}) started."|tee -a ${LOG}
echo "MAILTO=${MAILTO}"
echo "USDUSER=${USDUSER}"
echo "USDREPLYTO=${USDREPLYTO}"
echo "USDMAIL=${USDMAIL}"

for FILER in `cat ${FILERS}|grep -v \^#|grep ${FILTER}|awk -F\; '{print $1}'|sort`
do
  echo "${FILER} ..." 
  # PINGable?
  ${PING} -c 1 ${FILER} 1> /dev/null 2>&1
  EC=$?
  if [ ${EC} -gt 0 ]; then
    echo "`date` ${PGM}: Ping-Error to ${FILER} "|tee -a ${LOG}|tee -a ${ERR}
    MSGSTR="${MSGSTR}, Ping-Error:${FILER}"
  fi

# (1)Is RML/BMC pingable?
  ${PING} -c 1 ${FILER}-rlm 1> /dev/null 2>&1
  EC=$?
  if [ ${EC} -gt 0 ]; then
    # Check if -rlm is configured. If not (grep=0), then ERR is not true
    grep 'unknown host' ${TMP}
    EC=$?
    if [ ${EC} -gt 0 ]; then
      echo "`date` ${PGM}: Ping-Error to ${FILER}-rlm (RLM/BMC/console) "|tee -a ${LOG}|tee -a ${ERR}
      MSGSTR="${MSGSTR}, Ping-Error:${FILER}-rlm"
      RLMSTATUS="`${SSH} -n ${FILER} rlm status 2> /dev/null `" 
      if [ "${RLMSTATUS}" = "" ]; then
        echo "`date` ${PGM}: No RLMSTATUS. Maybe no RLM/BMC/console available"|tee -a ${LOG}|tee -a ${ERR}
        MSGSTR="${MSGSTR}, No RLMSTATUS"
      else
        ${SSH} ${FILER} rlm status|tee -a ${LOG}|tee -a ${ERR} 2> /dev/null
      fi
    fi
  fi

# (2)Check if filer-hostname is same
  FHOSTNAME="`${SSH} -n ${FILER} hostname`"
  if [ "${FILER}" != "${FHOSTNAME}" ]; then
# Check (2nd time) if uppercase
    ${SSH} -n ${FILER} hostname > ${TMP}
    ANSW="`grep -i ${FILER} ${TMP}`"
    if [ ${ANSW} = "" ]; then
      echo "`date` ${PGM}: Filername (${FILER}) <> hostname (${FHOSTNAME})"|tee -a ${LOG}|tee -a ${ERR}
      MSGSTR="${MSGSTR}, ${FILER}<>${FHOSTNAME}"
    fi
  fi

# (3)Check if ROOTVOL can be mounted
  ROOTVOL="`${SSH} -n ${FILER} vol status|grep ' root'|awk '{print $1}'`" 
  ETCDIR="/filers/${FILER}/${ROOTVOL}/etc/" 
# Test is DIR exsists. Otherwise (if not) script will stop
  if [ -d ${ETCDIR} ]; then
    cd ${ETCDIR} 2> /dev/null
    EC=$?
    if [ ${EC} -gt 0 ]; then
      echo "`date` ${PGM}: Error at ${FILER} changing to ${ETCDIR}"|tee -a ${LOG}|tee -a ${ERR}
      MSGSTR="${MSGSTR}, ERROR at cd ${FILER}:${ETCDIR}"
    fi
  else
    echo "`date` ${PGM}: Warning for ${FILER} no dir ${ETCDIR} found"|tee -a ${LOG}
  fi

# (4)Check autosupport is ON
  ASUP="`${SSH} -n ${FILER} options autosupport.enable|awk '{print $2}'`"
  if [ "${ASUP}" = "off" ]; then
    echo "`date` ${PGM}: Error at ${FILER} autosupport.enable is OFF"|tee -a ${LOG}|tee -a ${ERR}
      MSGSTR="${MSGSTR}, ${FILER} autosupport.enable is OFF}"
  fi

# (5)Check filer-time with managing host
  HOSTDATE="`date -u|awk '{print $1$2$3$6}'`"
  HOSTTIME="`date -u|awk '{print $4}'|awk -F\: '{print $1$2}'`"
  HOSTTIME="`date -u +%H%M`"
  FILERDATE="`${SSH} -n ${FILER} date -u|awk '{print $1$2$3$6}'`"
  FILERTIME="`${SSH} -n ${FILER} date -u|awk '{print $4}'|awk -F\: '{print $1$2}'`"
echo "Check filer time: ${HOSTDATE} ${HOSTTIME} = ${FILERDATE} ${FILERTIME} "
  if [ "${HOSTTIME}" != "${FILERTIME}" ] || [ "${HOSTDATE}" != "${FILERDATE}" ]; then
  #   time <> same
  # Check again. Can be at minute change
    sleep 5
    HOSTDATE="`date -u|awk '{print $1$2$3$6}'`"
    HOSTTIME="`date -u|awk '{print $4}'|awk -F\: '{print $1$2}'`"
    FILERDATE="`${SSH} -n ${FILER} date -u|awk '{print $1$2$3$6}'`"
    FILERTIME="`${SSH} -n ${FILER} date -u|awk '{print $4}'|awk -F\: '{print $1$2}'`"
echo "  2nd  ${HOSTDATE} ${HOSTTIME} = ${FILERDATE} ${FILERTIME} "
    if [ "${HOSTTIME}" != "${FILERTIME}" ] || [ "${HOSTDATE}" != "${FILERDATE}" ]; then
      echo "`date` ${PGM}: Error for ${FILER}; hosttime(`date -u`) NOT same with filer-time(` ${SSH} -n ${FILER} date -u`) " |tee -a ${LOG}|tee -a ${ERR}
      MSGSTR="${MSGSTR}, Time-sync-ERR:${FILER}"
    fi  # 2nd [ "${HOSTTIME}" != "${FILERTIME}" ] || [ "${HOSTDATE}" != "${FILERDATE}" ]
# When not in sync, USD+SMS need to be send. =CRITICAL
    NAGIOSSEV="CRITICAL"
    /usr/bin/logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} "${MSGSTR}"
    echo "`date` ${PGM}: logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} CI=${FILER} ${MSGSTR}" | tee -a ${LOG}
    grep NOTIFY ${MESSAGESFILE} | tail
    # When send to USD, MSGSTR can be cleared
    MSGSTR=""
  fi  # 1st [ "${HOSTTIME}" != "${FILERTIME}" ] || [ "${HOSTDATE}" != "${FILERDATE}" ]

  if [ ${LOGGER} ]; then
# NagiosSEV
# ------
# MINOR         Nagios-WARNING & SEV.3
# WARNING       Nagios-WARNING & SEV.3
# CRITICAL      Nagios-CRITICAL & SEV.2
# FATAL         Nagios-CRITICAL & SEV.2
# logger -t NOTIFY FSOD-SUPPORT <Nagios-SEV> "Text"
    if [ "${MSGSTR}" != "" ]; then
      NAGIOSSEV="WARNING"
##      /usr/bin/logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} CI=${FILER} "${MSGSTR}"
      echo "`date` ${PGM}: logger -t NOTIFY FSOD-SUPPORT ${NAGIOSSEV} CI=${FILER} ${MSGSTR}" | tee -a ${LOG}
      grep NOTIFY ${MESSAGESFILE} | tail
      # When send to USD, MSGSTR can be cleared
      MSGSTR=""
    fi  # [ {${MSGSTR} != "" ]
  fi  # [ ${LOGGER} ]

  # If ${ERR} (error msg), then mail / USD / LOGGER
  if [ -s ${ERR} ]; then
    if [ ${MAIL} ]; then
      cat ${ERR} | mailx -s ":${HOSTNAME}: Warning daily filer check [${PGM} v${VERSION}]" ${MAILTO}
      echo "`date` ${PGM}: Mailed to ${MAILTO}" | tee -a ${LOG}
      cp /dev/null ${ERR} 
    fi  # [ ${MAIL} ]

    if [ ${USD} ]; then
      echo "@REQUESTTYPE: Incident" > ${USDMSG}
      echo "@REQUESTAREA: NL.Storage.StorageOnDemand" >> ${USDMSG}
      echo "@SEVERITY: 4" >> ${USDMSG}
      echo "@CI: ${FILER}" >> ${USDMSG}
      echo "@ORGANISATION: AtosOrigin.Netherlands" >> ${USDMSG}
      echo "@GROUP: NL.Storage.FSOD" >> ${USDMSG}
      echo "@REPLYTO: ${USDREPLYTO}" >> ${USDMSG}
      echo "@PROXYUSER: ${USDUSER}"  >> ${USDMSG}
      cat ${ERR} >> ${USDMSG}
      echo "[${PGM} v${VERSION}]" >> ${USDMSG}
      echo "@ENDOFUSDMESSAGE" >> ${USDMSG}
      cat ${USDMSG} | mailx -s "${USDUSER}: Warning daily filer check." ${USDMAIL}
      echo "`date` ${PGM}: Mailed to USD (${USDMAIL})" | tee -a ${LOG}
      cp /dev/null ${ERR}
    fi  # [ ${USD} ]
  fi  # [ -s ${ERR} ]; then

done  # for FILER .. do

rm ${ERR} ${TMP} ${USDMSG}
echo "`date` ${PGM} finished." | tee -a $LOG
exit 0