#!/bin/sh
# File	: cdot_rpt_sv_cnt.sh
# By	: Maarten.deBoer@Atos.net, 150130
# Subject	: Script to collect CPU% for cDOT
#(0.2)	: Mod graphite
#(0.3)	: Mod's; remove CPU, Add SM & SV, Added SSHCMD
#(0.4)	: Start using TMP-file
PGM="`basename $0|cut -d\. -f1`"
VER="0.4"
TMP="/tmp/${PGM}.$$"
HOSTNAME="`hostname|cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
MAIL=""
MAILTO="maarten.deboer@atos.net"
FILTER="[?]*"
DATE="`date +%Y-%m-%d`"
DATI="`date +%Y-%m-%d' '%H:%M`"
DATASM="${HOME}/data/out/${PGM}-sm_${DATE}.csv"
DATASV="${HOME}/data/out/${PGM}-sv_${DATE}.csv"
#DATACPU="${HOME}/data/out/${PGM}-cpu_${DATE}.csv"
CLUSTERS="${HOME}/etc/clusters"
NC="${HOME}/bin/nc"
GRAPHITESRV="10.192.109.224 2003"
GRAPHITE=""
COUNTRY="nl"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VERSION}"
  echo "  options           :"
  echo "    -f              : Filter filername (${FILTER})"
  echo "    -g | --graphite : send to Graphite"
  echo "    -h | --help     : this help"
  echo "    -m | --mail     : do send Mail (only)"
  echo "    --mailto        : change MAILTO address & do send mail (${MAILTO})"
  echo "    -V              : Version"
  echo "    -x              : set -x"
}

# Check options
while [ ${#} -gt 0 ]
  do
  case ${1} in
    -f) FILTER=${2}; shift ;;
    -g | --graphite) GRAPHITE="1" ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    --mailto) MAILTO=${2}; MAIL=1; shift ;;
    -V) echo "${PGM}: v${VERSION}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "CLUSTERS=${CLUSTERS}"
echo "GRAPHITE=${GRAPHITE}"
echo "MAIL=${MAIL}"
echo "MAILTO=${MAILTO}"
sleep 1

touch ${TMP}

if [ ${MAIL} ]; then
  date | mailx -a ${DATASV} -s "@${HOSTNAME}: cDOT SnapVault report [${PGM} v${VER}]" ${MAILTO}
  date | mailx -a ${DATASM} -s "@${HOSTNAME}: cDOT SnapMirror report [${PGM} v${VER}]" ${MAILTO}
#  date | mailx -a ${DATACPU} -s "@${HOSTNAME}: cDOT CPU report [${PGM} v${VER}]" ${MAILTO}
  echo "`date` mailed to ${MAILTO}"|tee -a ${LOG}
else

  touch ${DATASV}
  touch ${DATASM}
#  touch ${DATACPU}

  SVCSVSTR="${DATI}"
  SMCSVSTR="${DATI}"
#  CPULOADSTR="${DATI};"
  HEADER="DateTime; "
  for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${FILTER}"`
  do
    echo "  ${CLUSTER} ... "
# Because getting "snapmirror" info can cost some time, do this once in TMP-file. And then extract info from TMP-file
    SSHCMD ${CLUSTER} 'snapmirror show' > ${TMP}

    HEADER="${HEADER} ${CLUSTER}; ;"

    SVCNT="`cat ${TMP}|grep nl|grep ' XDP '|grep Snapmirrored|grep -v Idle|wc -l`"
    SVTTLCNT="`cat ${TMP}|grep nl|grep ' XDP '|grep Snapmirrored|wc -l`"
    SVCSVSTR="${SVCSVSTR};${SVCNT};${SVTTLCNT}"

    SMCNT="`cat ${TMP}|grep nl|grep ' DP '|grep Snapmirrored|grep -v Idle|wc -l`"
    SMTTLCNT="`cat ${TMP}|grep nl|grep ' DP '|grep Snapmirrored|wc -l`"
    SMCSVSTR="${SMCSVSTR};${SMCNT};${SMTTLCNT}"

    echo "SVCSVSTR=${SVCSVSTR}"
    echo "SMCSVSTR=${SMCSVSTR}"

#    echo "${CPULOAD} - ${SMCNT} / ${SVCNT}"
    if [ ${GRAPHITE} ]; then
      DATE_S="`date +%s`"
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapvault.cnt ${SVCNT} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapvault.total-cnt ${SVTTLCNT} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapmirror.cnt ${SMCNT} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapmirror.total-cnt ${SMTTLCNT} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "`date` send to Graphite (${GRAPHITESRV}) for ${COUNTRY}.${CLUSTER}"|tee -a ${LOG}
    fi  # GRAPHITE

  done  # CLUSTER

  HR="`date +%H`"
  if [ ${HR} -eq 0 ] || [ ${HR} -eq 12 ]; then
    echo ${HEADER} |tee -a ${DATASV}
    echo ${HEADER} |tee -a ${DATASM}
#    echo ${HEADER} |tee -a ${DATACPU}
  fi  # HR
  echo ${SVCSVSTR} |tee -a ${DATASV}
  echo ${SMCSVSTR} |tee -a ${DATASM}
#  echo ${CPULOADSTR} |tee -a ${DATACPU}
  
fi  # MAIL

rm ${TMP}
echo "`date` finished."|tee -a ${LOG}
exit 0