#!/bin/sh
# File	: cdot_rpt_sm_dura.sh
# By	: Maarten.deBoer@Atos.net, 171123
# Subject	: Script to report (to Graphite) SnapMirror Durations
#(0.2),171123	: Add to CSV
#(0.3),171123	: Add SV speed (KB/min)
PGM="`basename $0|cut -d\. -f1`"
VER="0.3"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
HOSTNAME="`hostname|cut -d\. -f1`"
CLUSTERS="${HOME}/etc/clusters"
MAIL=""
MAILTO="maarten.deboer@atos.net"
FILTER="[?]*"
DATE="`date +%Y-%m-%d`"
DATI="`date +%Y-%m-%d' '%H:%M`"
DATASV="${HOME}/data/out/${PGM}-sv_${DATE}.csv"
DATASM="${HOME}/data/out/${PGM}-sm_${DATE}.csv"

NC="${HOME}/bin/nc"
GRAPHITESRV="10.192.109.224 2003"
GRAPHITE=""
COUNTRY="nl"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}


USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VERSION}"
  echo "  options           :"
  echo "    -f              : Filter filername (${FILTER})"
  echo "    -g | --graphite : send to Graphite"
  echo "    -h | --help     : this help"
  echo "    -m | --mail     : do send Mail (only)"
  echo "    --mailto        : change MAILTO address & do send mail (${MAILTO})"
  echo "    -V              : Version"
  echo "    -x              : set -x"
}

# Check options
while [ ${#} -gt 0 ]
  do
  case ${1} in
    -f) FILTER=${2}; shift ;;
    -g | --graphite) GRAPHITE="1" ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    --mailto) MAILTO=${2}; MAIL=1; shift ;;
    -V) echo "${PGM}: v${VERSION}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done

# MAIN

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "CLUSTERS=${CLUSTERS}"
echo "GRAPHITE=${GRAPHITE}"
echo "MAIL=${MAIL}"
echo "MAILTO=${MAILTO}"
sleep 1

touch ${TMP}
echo "0" > ${TMP}.2

if [ ${MAIL} ]; then
  date | mailx -a ${DATASV} -s "@${HOSTNAME}: cDOT SnapVault report [${PGM} v${VER}]" ${MAILTO}
#  date | mailx -a ${DATASM} -s "@${HOSTNAME}: cDOT SnapMirror report [${PGM} v${VER}]" ${MAILTO}
  echo "`date` mailed to ${MAILTO}"|tee -a ${LOG}
else
  touch ${DATASV}
  touch ${DATASM}
  SVCSVSTR="${DATI}"
  SMCSVSTR="${DATI}"
  HEADER="DateTime; "
  for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${FILTER}"`
  do
    echo "  ${CLUSTER} ... "
    HEADER="${HEADER};${CLUSTER} [KB]; ${CLUSTER} [min]; ${CLUSTER} [KB/min]"
# 'snapmirror show -type XDP -last-transfer-end-timestamp "11/23 11:00:00".."11/23 17:00:00" -field last-transfer-duration,last-transfer-end-timestamp, last-transfer-size' 
    HR="`date +%H --date="last hour"`"
    DAY="`date +%d`"
    MON="`date +%m`"
    echo "  For ${MON}/${DAY} ${HR}:00:00 ..."
  
    SSHCMD ${CLUSTER} "set -unit KB -showseparator \";\" ; snapmirror show -type XDP -last-transfer-end-timestamp \"${MON}/${DAY} ${HR}:00:00\"..\"${MON}/${DAY} ${HR}:59:59\" -field last-transfer-duration,last-transfer-end-timestamp, last-transfer-size" > ${TMP}

# cat ${TMP}

    TTLSIZE=0
    echo "0" > ${TMP}.2
    cat ${TMP}| grep nl|awk -F\; '{print $3}'|sed 's/KB//g'| while read SIZE REST
    do
      TTLSIZE=`expr ${TTLSIZE} + ${SIZE}`
#      echo "${SIZE} => ${TTLSIZE}"
    echo "${TTLSIZE}" > ${TMP}.2
    done
    TTLSIZE=`cat ${TMP}.2`
    echo "${TTLSIZE}KB"
    SVCSVSTR="${SVCSVSTR};${TTLSIZE}"

    TTLDURA=0
    TTLSEC=0
    echo "0" > ${TMP}.2
    cat ${TMP}| grep nl|awk -F\; '{print $4}'| while read DURA REST
    do
      HR=`echo ${DURA}|awk -F\: '{print $1}'`
      MIN=`echo ${DURA}|awk -F\: '{print $2}'`
      SEC=`echo ${DURA}|awk -F\: '{print $3}'`
#      echo "|${DURA}| ${HR} ${MIN} ${SEC}"

      HRMIN=`expr ${HR} \* 60`
      TTLDURA=`expr ${TTLDURA} + ${HRMIN} + ${MIN}`
      TTLSEC=`expr ${TTLSEC} + ${SEC}`
# Count Sec's separatly. And add 1 min when more then 60 sec's
      if [ ${TTLSEC} -gt 60 ]; then
        TTLDURA=`expr ${TTLDURA} + 1`
        TTLSEC=`expr ${TTLSEC} - 60`
      fi
#      echo "${DURA} => ${TTLDURA}"
      echo "${TTLDURA}" > ${TMP}.2
    done
    TTLDURA=`cat ${TMP}.2`
    echo "${TTLDURA}min"
    SVCSVSTR="${SVCSVSTR};${TTLDURA}"

    SVSPEED=0
    if [ ${TTLDURA} -gt 0 ]; then
      SVSPEED=`expr ${TTLSIZE} / ${TTLDURA}`
      echo "SVSPEED=${SVSPEED} KB/min"
      SVCSVSTR="${SVCSVSTR};${SVSPEED}"
    fi

    if [ ${GRAPHITE} ]; then
      DATE_S="`date +%s`"
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapvault.total-size ${TTLSIZE} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapvault.total-duration ${TTLDURA} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapvault.speed ${SVSPEED} ${DATE_S}"|${NC} ${GRAPHITESRV}
#    echo "netapp.cluster.${COUNTRY}.${CLUSTER}.snapmirror.duration ${SMDURA} ${DATE_S}"|${NC} ${GRAPHITESRV}
      echo "`date` send to Graphite (${GRAPHITESRV}) for ${COUNTRY}.${CLUSTER}"|tee -a ${LOG}
    fi  # GRAPHITE

  done  # CLUSTER


  if [ ${HR} -eq 0 ] || [ ${HR} -eq 12 ]; then
    echo ${HEADER} |tee -a ${DATASV}
    echo ${HEADER} |tee -a ${DATASM}
  fi  # HR
  echo ${SVCSVSTR} |tee -a ${DATASV}
  echo ${SMCSVSTR} |tee -a ${DATASM}

fi  # MAIL

rm ${TMP} ${TMP}.2
echo "`date` finished."|tee -a ${LOG}
exit 0