#!/bin/sh
# File	: cdot_chk_autosizing.sh
# By	: Maarten.deBoer@atos.net, 141110, 200812
# Subject	: Script to check NetApp's cDOT autosizing.
#(0.2),200812	: Added check at < CHK_PCT_USED
#(0.3),200813	: Add "quota report" to TMP-file. To speed-up
#(0.4),200813	: Some add's. When SUM_QUOTA=0 (probably TYPE=DP), then NO ALERT
#
PGM="`basename $0|cut -d\. -f1`"
VER="0.4"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
MAILTO="maarten.deboer@atos.net"
DATE="`date +%Y-%m-%d`"
CFILTER="[?]*"
SFILTER="[?]*"
VFILTER="[?]*"
MAIL=""
CHK_PCT_USED=75
CSV="/tmp/${PGM}.csv"

PREFIX="nl"
CLUSTERS="${HOME}/etc/clusters"

USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VER}"
  echo "  options           :"
  echo "    -c | --clusters : Cluster filter (${CFILTER})"
  echo "    -s | --svm      : Svm filter (${SFILTER})"
  echo "    -v | --vol      : Volume filter (${VFILTER})"
  echo "    -e              : Etc/clusters file (${CLUSTERS})"
  echo "    -h | --help     : this help"
  echo "    -m | --mail     : do send Mail"
  echo "    --mailto        : change MAILTO address & do send mail (${MAILTO})"
  echo "    -V              : Version"
  echo "    -x              : set -x"
}

# Check options
while [ ${#} -gt 0 ]
  do
  case ${1} in
    -c | --clusters) CFILTER=${2}; shift ;;
    -s | --svm) SFILTER=${2}; shift ;;
    -v | --vol) VFILTER=${2}; shift ;;
    -e) CLUSTERS=${2}; shift ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    --mailto) MAILTO=${2}; MAIL=1; shift ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}


# MAIN
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "CLUSTERS=${CLUSTERS}"
echo "CSV=${CSV}"
echo "CFILTER=${CFILTER}"
echo "SFILTER=${SFILTER}"
echo "VFILTER=${VFILTER}"
echo "MAIL=${MAIL}"
echo "MAILTO=${MAILTO}"
echo "PREFIX=${PREFIX}"

sleep 1
echo "  ${PGM}: CFILTER=${CFILTER} SFILTER=${SFILTER} VFILTER=${VFILTER} MAIL=${MAIL} MAILTO=${MAILTO} " >> ${LOG}

touch ${TMP} ${TMP}.SUM_QUOTA ${TMP}.ALERT
echo "# vserver;volume;aggregate;size [GB];used[GB];percent-used [%];max-autosize [GB];min-autosize [GB];autosize-grow-threshold-percent [%];autosize-shrink-threshold-percent [%];autosize-mode;type;sum_of_quota;" > ${TMP}.csv

for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep ${CFILTER}`
do
  echo "  ${CLUSTER} ..." 
  SSHCMD ${CLUSTER} 'vserver show -operational-state running -field vserver'|grep "${PREFIX}"|grep "${SFILTER}"|while read VSERVER REST
  do
    echo "  ${CLUSTER}:${VSERVER} ..."
    SSHCMD ${CLUSTER} "volume show -vserver ${VSERVER} -field volume"|grep ${VSERVER}|grep "${VFILTER}"|awk '{print $2}'|while read VOLUME REST
    do
      echo "  ${CLUSTER}:${VSERVER}:${VOLUME} ..."

      ANSW=`SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ; volume show -vserver ${VSERVER} -volume ${VOLUME} -field aggregate,size,used,percent-used,autosize-grow-threshold-percent,autosize-shrink-threshold-percent,autosize-mode,min-autosize,max-autosize,min-autosize,type"|grep ${PREFIX}|sed 's/GB//g'|sed 's/%//g' `
#      echo "${ANSW}"
      AGGR=`echo "${ANSW}"|awk -F\; '{print $3}' `
      SIZE=`echo "${ANSW}"|awk -F\; '{print $4}' `
      USED=`echo "${ANSW}"|awk -F\; '{print $5}' `
      PCT_USED=`echo "${ANSW}"|awk -F\; '{print $6}'`
      MAX_AS=`echo "${ANSW}"|awk -F\; '{print $7}' `
      MIN_AS=`echo "${ANSW}"|awk -F\; '{print $8}' `
      AS_GROW_PCT=`echo "${ANSW}"|awk -F\; '{print $9}'`
      AS_SHRINK_PCT=`echo "${ANSW}"|awk -F\; '{print $10}'`
      AS_MODE=`echo "${ANSW}"|awk -F\; '{print $11}' `
      TYPE=`echo "${ANSW}"|awk -F\; '{print $12}' `

# SUM Quota
      SUM_QUOTA=0
# Added: Using TMP-file for Quota-report /volume (to speed-up)
      SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ; quota report -vserver ${VSERVER} -volume ${VOLUME} -field vserver,volume,tree,disk-limit"|grep ${VSERVER} > ${TMP}
      SSHCMD ${CLUSTER} "qtree show -vserver ${VSERVER} -volume ${VOLUME} -fields qtree"|grep ${VSERVER}|awk '{print $3}'|while read QTREE REST
      do
        if [ "${QTREE}" != "\"\"" ]; then
          QUOTA=`grep ${QTREE} ${TMP}|grep ${VSERVER}|awk -F\; '{print $5}'|sed 's/GB//g'`
#echo "QUOTA=${QUOTA}"

          if [ "${QUOTA}" = "" ]; then
            QUOTA=0
          fi
          if [ ${QUOTA} -gt 0 ]; then
            SUM_QUOTA=`expr ${SUM_QUOTA} + ${QUOTA}`
          fi
        fi  # QTREE <> ""
        echo "        ${QTREE}=${QUOTA}GB"
# To get the value out site the subshell-loop
        echo ${SUM_QUOTA} > ${TMP}.SUM_QUOTA
      done  # QTREE
      SUM_QUOTA=`cat ${TMP}.SUM_QUOTA`

# ALERTing
# Check for used% lower then threshold (CHK_PCT_USED) at volumes > 500GB
# AND SUM of the Quota < (vol.)SIZE
# Add: When SUM_QUOTA=0 (probably TYPE=DP), then NO ALERT
      if [ ${SUM_QUOTA} -gt 0 ] && [ ${SIZE} -gt ${SUM_QUOTA} ] && [ ${SIZE} -gt 500 ] && [ ${PCT_USED} -lt ${CHK_PCT_USED} ]; then
        echo "** ALERT ** : ${VOLUME} = ${PCT_USED}% used and volume(-SIZE) > 500GB and volume(-SIZE)=${SIZE}GB > SUM_QUOTA=${SUM_QUOTA}GB (TYPE=${TYPE} MIN_AS=${MIN_AS} MAX_AS=${MAX_AS})"|tee -a ${LOG}|tee -a ${TMP}.ALERT

        echo "${VSERVER};${VOLUME};${AGGR};${SIZE};${USED};${PCT_USED};${MAX_AS};${MIN_AS};${AS_GROW_PCT};${AS_SHRINK_PCT};${AS_MODE};${TYPE};${SUM_QUOTA};" >> ${TMP}.csv
      fi

      echo "  CLUSTER=${CLUSTER} VSERVER=${VSERVER} VOLUME=${VOLUME}"
      echo "  AGGR=${AGGR} SIZE=${SIZE}GB USED=${USED}GB PCT_USED=${PCT_USED}% TYPE=${TYPE}"
      echo "  MIN_AS=${MIN_AS}GB MAX_AS=${MAX_AS}GB AS_SHRINK_PCT=${AS_SHRINK_PCT}% AS_GROW_PCT=${AS_GROW_PCT}% AS_MODE=${AS_MODE}"
      echo "  SUM_QUOTA=${SUM_QUOTA}GB"

    done  # VOLUME

  done  # VSERVER

done  # CLUSTER

if [ -s ${TMP}.ALERT ] && [ ${MAIL} ]; then
  echo "" >> ${TMP}.ALERT
  echo "" >> ${TMP}.ALERT
  echo "#" | tee -a ${TMP}.ALERT >> ${TMP}.csv
  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" |tee -a ${TMP}.ALERT >> ${TMP}.csv
  cp ${TMP}.csv ${CSV}
  cat ${TMP}.ALERT | mailx -a ${CSV} -s ":${HOSTNAME}: Autosize alerting  [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
fi  #  MAIL

# Finish
rm ${TMP} ${TMP}.SUM_QUOTA ${TMP}.csv ${TMP}.ALERT
echo "`date` ${PGM} finished."|tee -a ${LOG}
exit 0