#!/bin/sh
# File	: cdot_chk_vservers.sh
# By	: Maarten.deBoer@atos.net, 220824
# Subject	: Script to check vservers with NO volumes
# set -x
#(0.2),220824	: Mod. all vserver (not only running)
PGM="`basename $0|cut -d\. -f1`"
VER="0.2"
TMP="/tmp/${TMP}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
MAILTO="maarten.deboer@atos.net"
MAIL=""
CLUSTERS="${HOME}/etc/clusters"
CFILTER="[?]*"
SFILTER="[?]*"
CSV="/tmp/${PGM}.csv"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 2
    /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 2
      /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 2
        /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} sleep 2 ..."
            sleep 2
            /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}

            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              echo "  EC=${EC} sleep 2 ..."
              sleep 2
              /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}

              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0

            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}
USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VER}"
  echo "  options          :"
  echo "    -c | --cluster : Cluster filter (${CFILTER})"
  echo "    -s | --svm     : Svm filter (${SFILTER})"
  echo "    -m | --mail    : Mail the logging"
  echo "    -h | --help    : this help"
  echo "    -V             : Version"
  echo "    -x             : set -x"
  echo "    --mailto       : set new MAILTO (${MAILTO})"
}
# Check options
while [ ${#} -gt 0 ]
do
  case ${1} in
    -c | --cluster) CFILTER="${2}"; shift ;;
    -s | --svm) SFILTER="${2}"; shift ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    --mailto) MAIL=1; MAILTO="${2}"; shift ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done  # while


# MAIN
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "CLUSTERS=${CLUSTERS}"
echo "MAILTO=${MAILTO}"
echo "MAIL=${MAIL}"
echo "CFILTER=${CFILTER}"
echo "SFILTER=${SFILTER}"
sleep 1


touch ${TMP}
cp /dev/null ${CSV}
for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"`
do
  echo "${CLUSTER};" | tee -a ${CSV}
  SSHCMD ${CLUSTER} 'vserver show -field vserver'|grep nlnafs|grep "${SFILTER}"|while read VSERVER REST
  do
    echo -n "  ${CLUSTER}/${VSERVER} ... "

    SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume show -vserver ${VSERVER} -field vserver,volume"|grep "${VSERVER}" > ${TMP}
    VOL_CNT=`cat ${TMP}|wc -l`
    echo " Volume count = ${VOL_CNT}"
    if [ ${VOL_CNT} -le 1 ]; then
      echo "    Volume count <= 1 (${VOL_CNT}):"
      cat ${TMP} | tee -a ${CSV}
    fi  # VOL_CNT <
  done  # VSERVER

done  # for CLUSTER

if [ ${MAIL} ]; then
  echo "`date` ${PGM}: File attached with vserver with <= 1 volumes. Filters: C=${CFILTER} S=${SFILTER}" | mailx -a ${CSV} -s ":${HOSTNAME}: vserver check [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
fi

# Finish
rm ${TMP} 
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0