
#!/bin/ksh
# File	: cdot_rpt_snap_backups.sh
# By	: Maarten.deBoer@Atosnet, 160226
# Subject	: To make a report on NAS(FSOD)backup (SnapShot & SnapVaults) at cDOT
# Initialize variabels                                                     #
#(0.2),220504	: Upd. mail part. Removed RHEL check
#(0.3),230727	: From cdot_make_backup_report.sh, added snapshot-policy,sched-snap-name,snapshot-count
#(0.4),230731	: Added at -field size,blocks,usedblocks
#(0.5),230802	: Renamed from cdot_make_snap_backup_report.sh
#(0.6),230802	: Add -state online
#(0.7),230810	: Added sshcmd.func
PGM="`basename $0|cut -d\. -f1`"
VER="0.7"
TMP="/tmp/${PGM}.$$"
HOSTNAME="`hostname|cut -d\. -f1`"
SSH="/usr/bin/ssh -n"
CLUSTERS="${HOME}/etc/clusters"
MAXLOGSIZE=1024   # In K's
LOG="${HOME}/log/${PGM}.log"
MAILTO="maarten.deboer@atos.net"
FILTER="[?]*"
CUSTCODE=""
DATE=`date '+%Y-%m-%d'`
MAIL=""

#--------------------------------------------------------------------------#
# Functions                                                                #
#--------------------------------------------------------------------------#
# File	: sshcmd.func
# VER="0.1", <node>-04
# VER="0.2", <node>-06
# VER="0.3", <node>-10
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.tmperr"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                sleep 1
                /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
                EC=${?}
                grep 'Connection refused' ${TMPERR}
                EC2=${?}
                if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                  sleep 1
                  /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
                  EC=${?}
                  grep 'Connection refused' ${TMPERR}
                  EC2=${?}
                  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                    sleep 1
                    /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
                    EC=${?}
                    grep 'Connection refused' ${TMPERR}
                    EC2=${?}
                    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                      sleep 1
                      /usr/bin/ssh -n ${1}-12 "${2}" 2> ${TMPERR}
                      EC=${?}
                      grep 'Connection refused' ${TMPERR}
                      EC2=${?}
                      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                        echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 .. -10 failed too."|tee -a ${LOG} 
                      fi 
                    fi
                  fi 
                fi 
              fi 
            fi  
          fi  
        fi  
      fi  
    fi  
  fi  # 
  rm ${TMPERR}
}


USAGE()
{
  echo "Usage: ${PGM} -c <cust.code> [<options>]"
  echo "  Version: ${VER}"
  echo "  options    :"
  echo "    -c       : Custom code"
  echo "    -f       : cluster Filter (${FILTER})"
  echo "    -h       : this help"
  echo "    -m       : do send mail"
  echo "    -V       : Version"
  echo "    -x       : set -x"
  echo "    --mail   : do send mail"
  echo "    --mailto : change MAILTO address & do send mail (${MAILTO})"
  echo "    --help   : this help"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -f) FILTER=${2}; shift ;;
    -c) CUSTCODE=${2}; shift ;;
    -m | --mail) MAIL=1 ;;
    --mailto) MAILTO="${2}"; MAIL=1; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    -u | --usd) USD=1;;
    *)  echo "`date` Option $1 not known."|tee -a ${LOG}; USAGE; exit 1 ;;
  esac
    shift
done

if [ -z "${CUSTCODE}" ]
then
  echo "`date` No (part of) customer code."|tee -a ${LOG}
  USAGE
  echo "`date` $0 (v${VER}) Exiting ..."|tee -a ${LOG}
  exit 1
fi

touch ${TMP}
CUSTOMER=`echo "${CUSTCODE}"|tr [:lower:] [:upper:]`
CSV="FSOD_cdot_snap_backup_report_${CUSTOMER}_${DATE}.csv"

#--------------------------------------------------------------------------#
#  LOG rotating
#--------------------------------------------------------------------------#
touch ${LOG}
# Check & move LOG-file if longer then max.
LOGSIZE=`du -ka $LOG | cut -f1`
if [ $LOGSIZE -ge $MAXLOGSIZE ]; then
  mv $LOG $LOG.old
  touch $LOG
fi
echo "`date` $0 (v${VER}) started."|tee -a ${LOG}

for CLUSTER in `cat ${CLUSTERS}|grep -v ^\#|grep ${FILTER}|awk -F\; '{print $1}'`
do
  echo "# ${CLUSTER};Job schedules /cluster ..."|tee -a ${TMP}
# Schedules /cluster
      SSHCMD ${CLUSTER} "set -showseparator \";\";job schedule show -fields name,type,description "|grep -v description >> ${TMP}

#  for VSERVER in `SSHCMD ${CLUSTER} 'vserver show -operational-state running -fields vserver'|grep nlnaf|grep ${CUSTCODE}|sort`
#  do
  SSHCMD ${CLUSTER} "vserver show -operational-state running -fields vserver"|grep nlnaf|grep ${CUSTCODE}|sort|while read VSERVER REST
  do
    echo "# ${CLUSTER}/${VSERVER};Volume snapshot policy /SVM ..."|tee -a ${TMP}
    SSHCMD ${CLUSTER} "set -showseparator \";\";volume snapshot policy show -vserver ${VSERVER} -fields policy,schedules,prefixes"|grep -v vserver >> ${TMP}

    echo "# ${CLUSTER}/${VSERVER};Snapvault (snapmirror-XDP) /SVM ..."|tee -a ${TMP}
    SSHCMD ${CLUSTER} "set -showseparator \";\";snapmirror show -vserver ${VSERVER} -type xdp -fields destination-path,schedule,policy-type,type,policy,status,exported-snapshot,total-progress "|grep -v source-path >> ${TMP}

# List of the snaps, config /volume
    SSHCMD ${CLUSTER} "volume show -vserver ${VSERVER} -state online -field volume"|grep ${CUSTCODE}|awk '{print $2}'|while read VOL
    do
      echo "# ${CLUSTER}/${VSERVER};${VOL};Snap Config /volume ..."|tee -a ${TMP}
      SSHCMD ${CLUSTER} "set -showseparator \";\";volume show -vserver ${VSERVER} -volume ${VOL} -field vserver,volume,snapshot-policy,sched-snap-name,snapshot-count"|grep -v vserver >> ${TMP}

      echo "# ${CLUSTER}/${VSERVER};${VOL};Snaps /volume ..."|tee -a ${TMP}
      SSHCMD ${CLUSTER} "set -showseparator \";\" -unit GB;snap show -vserver ${VSERVER} -volume ${VOL} -field vserver,volume,create-time,snapmirror-label,size,blocks,usedblocks"|grep -v vserver >> ${TMP}

    done  # ${VOL}
  done  # for VSERVER
done   # for CLUSTER in 

echo "# ${PGM} v${VER}"|tee -a ${TMP}
echo "# With: (part)customer code=${CUSTCODE}, (cluster)FILTER=${FILTER}, MAILTO=${MAILTO} "|tee -a ${TMP}
echo "# Made at `date` at ${HOSTNAME} "|tee -a ${TMP}

SUBJECT="@${HOSTNAME}: FSOD backup report ${CUSTOMER} ${DATE} [${PGM} v${VER}]"

if [ ${MAIL} ]; then
  cp ${TMP} /tmp/${CSV}
  echo "${PGM}: [${PGM} v${VER}] at `date`"|mailx -s "@${HOSTNAME}: FSOD cDOT snap backup report ${CUSTOMER} ${DATE} [${PGM} v${VER}]" -a /tmp/${CSV} ${MAILTO}
  echo "  Mailed ${CSV} to ${MAILTO} (${SUBJECT}) "|tee -a ${LOG}
  rm /tmp/${CSV}
fi

rm ${TMP} 
echo "`date` ${PGM} v${VER} finished."|tee -a $LOG
exit 0

