#!/bin/ksh
# File  : cdot_chk_aggr_cap.sh
# By    : Maarten de Boer, 220707
# Subject       : Script to ckeck aggr capacity (usage)
# set -x
#(0.2),220808	: Mod. (n volumes)
#(0.3),220815	: Mod.; when no alerts. Msg is "NO alerts found".
#(0.4),231120	: Add volume sizes
#(0.5),231221	: Added SHOW_VOLS, -v, Mod. ALERT_MSG
PGM="`basename $0|cut -d\. -f1`"
VER="0.5"
TMP="/tmp/${PGM}.$$"
CLUSTERS="${HOME}/etc/clusters"
SSH="/usr/bin/ssh"
HOSTNAME="`hostname | cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
AFILTER="[?]*"
CFILTER="[?]*"
SFILTER="[?]*"
MAIL=""
MAILTO="maarten.deboer@atos.net"
CSV="/tmp/${PGM}.csv"
DATI=`date +%Y-%m-%d-%H-%M-%S`
MAX_PERC=85
ALERT_MSG=""
SHOW_VOLS=""

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options          :"
  echo "    -a|--aggr      : filter for AGGRname (${AFILTER})"
  echo "    -c|--cluster   : filter for Clustername (${CFILTER})"
  echo "    -h|--help      : this Help"
  echo "    -m|--mail      : do send Mail"
  echo "    -p|--perc      : max Percentage (${MAX_PERC})"
  echo "    -s|--svm       : filter for SVMname (${SFILTER})"
  echo "    -v|--show_vols : show Volumes (in output)"
  echo "    -V             : show Version"
  echo "    -x             : set -x"
  echo "    --mailto       : change MAILTO address & do send mail (${MAILTO})"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -a | --aggr) AFILTER="${2}"; shift ;;
    -c | --cluster) CFILTER="${2}"; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -m | --mail) MAIL=1 ;;
    -v | --show_vols) SHOW_VOLS=1 ;;
    -p | --perc) MAX_PERC=${2}; shift ;;
    -s | --svm) SFILTER="${2}"; shift ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    --mailto) MAIL=1; MAILTO="${2}"; shift ;;
    *)  echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;;
  esac
    shift
done  # case

echo "`date` ${PGM} v${VER} started"|tee -a ${LOG}

echo "AFILTER=${AFILTER}"
echo "ALERT_MSG=${ALERT_MSG}"
echo "CFILTER=${CFILTER}"
echo "CLUSTERS=${CLUSTERS}"
echo "CSV=${CSV}"
echo "DATI=${DATI}"
echo "MAIL=${MAIL}"
echo "MAILTO=${MAILTO}"
echo "MAX_PERC=${MAX_PERC}"
echo "SFILTER=${SFILTER}"
echo "SHOW_VOLS=${SHOW_VOLS}"
sleep 1

touch ${TMP}

echo "# Aggregate;Node;Size [GB];Used Size [GB];Physical Used Percentage [%];Number Of Volumes;Vserver;Volume Name;Volume Size [GB];Qtree name;" > ${CSV}

cat "${CLUSTERS}"|grep -v \^#|awk -F\; '{print $1}'|sort|grep "${CFILTER}"|while read CLUSTER
do
  echo "  ${CLUSTER} ..."
#|tee -a ${TMP}
  SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ;storage aggr show -state online -has-mroot false -field node,size,volcount,physical-used-percent,used"|grep ${CLUSTER}|while read LINE
  do
#    echo "  ${LINE}"
    AGGR=`echo ${LINE}|awk -F\; '{print $1}'|grep "${AFILTER}"`
    NODE=`echo ${LINE}|awk -F\; '{print $2}'`
    SIZE=`echo ${LINE}|awk -F\; '{print $4}'|sed 's/GB//g'`
    USED=`echo ${LINE}|awk -F\; '{print $5}'|sed 's/GB//g'`
    NUM_VOLUMES=`echo ${LINE}|awk -F\; '{print $6}'`
    USED_PERC=`echo ${LINE}|awk -F\; '{print $7}'|sed 's/%//g'`
    echo "${AGGR};${NODE};${SIZE};${USED};${USED_PERC};${NUM_VOLUMES};"|tee -a ${CSV}
    ALERT_MSG=""
    if [ ${USED_PERC} -ge ${MAX_PERC} ]; then
      echo "  ALERT (>=${MAX_PERC}%) on ${CLUSTER} for ${USED_PERC}% on ${AGGR} at ${NODE} : ${USED}GB / ${SIZE}GB (${NUM_VOLUMES} volumes)."|tee -a ${TMP}
      ALERT_MSG=1

# Getting volumes of this AGGR, when SHOW_VOLS
      if [ ${SHOW_VOLS} ]; then

      SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ;volume show -aggregate ${AGGR} -state online -field node,volume,size"|grep ${CLUSTER}|while read LINE2
      do
#        echo "  LINE2=${LINE2}"
        VSERVER=`echo ${LINE2}|awk -F\; '{print $1}'|grep "${SFILTER}"`
        VOLUME=`echo ${LINE2}|awk -F\; '{print $2}'`
        SIZE=`echo ${LINE2}|awk -F\; '{print $3}'|sed 's/GB//g'`
        NODE=`echo ${LINE2}|awk -F\; '{print $4}'`
        echo "    ${VOLUME}"
#        echo "${VSERVER};${VOLUME};${NODE};"
        SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ;qtree show -volume ${VOLUME} -field qtree,volume"|grep ${VSERVER}|while read LINE3
        do
#          echo "      ${LINE3}"
          QTREE=`echo ${LINE3}|awk -F\; '{print $3}'`
          if [ "${QTREE}" != '""' ]; then
            echo "${AGGR};${NODE}; ; ; ; ;${VSERVER};${VOLUME};${SIZE};${QTREE};"|tee -a ${CSV}
          fi   # <> ""

        done  # LINE3 (qtree show)

      done  # LINE2 (volume show)

      fi  #SHOW_VOLS

    fi  # {USED_PERC} -gt ${MAX_PERC

  done  # LINE (storage aggr show )

  if [ "${ALERT_MSG}" = "" ]; then
    echo "    NO alert (AGGR's >= ${MAX_PERC}%) found for ${CLUSTER}."|tee -a ${TMP}
  fi  # ALERT_MSG

done  # CLUSTER


if [ ${MAIL} ]; then
  echo "---" >> ${TMP}
  echo "`date` ${PGM} v${VER} at `hostname`" >> ${TMP}
  cat ${TMP} | mailx -a ${CSV} -s ":${HOSTNAME}: ALERTs on AGGR usage (> ${MAX_PERC}%) [${PGM} v${VER}]" ${MAILTO} 
  echo "  Mailed (${CSV}) to ${MAILTO}"|tee -a ${LOG}
fi  # MAIL

rm ${TMP} ${CSV}
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0