#!/bin/sh
# File	: cdot_report_autosizing.sh
# By	: Maarten.deBoer@atos.net, 141110
# Subject	: Script to report information from cDOT filers. 
#		  About autosizing
#(0.2)	Mod.
#(0.3),160713	: Mods like SSHCMD
#(0.4),170614	: Added Quota report, VOLFILTER
#(0.5),180927	: Added snapshot-size
#(0.6),181003	: Added Snapshots reporting (large amount & size) SNAPRPT (-s) & QTRERPT (-q)
#(0.7),190403	: Changed name from cdot_report_autosizing.sh, Mod. mail-subject
#(0.8),200825	: Added DOALL, MAILMSG. Mod VFILTER, SFILTER, CFILTER, DISKLIMIT => QUOTA, SNAPRTP -> SNAP_RPT
PGM="`basename $0|cut -d\. -f1`"
VER="0.8"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
MAILTO="maarten.deboer@atos.net"
DATE="`date +%Y-%m-%d`"
CSV="FSOD_autosize-report_${DATE}.csv"
CFILTER="[?]*"
SFILTER="[?]*"
VFILTER="[?]*"
ERR="${HOME}/log/${PGM}.err"
MAIL=""
QTREERPT=""
SNAP_RPT=""
SNAPCSV="FSOD-Snap_autosize-report_${DATE}.csv"
DOALL=""
MAILMSG=${TMP}.mailmsg

PREFIX="nlnaf"

CLUSTERS="${HOME}/etc/clusters"
CUSTCODE=""

USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VER}"
  echo "  options          :"
  echo "    -c | --cluster : Clusterfilter (${CFILTER})"
  echo "    --svm          : SVMfilter (${SFILTER})"
  echo "    -v | --vol     : Volfilter (${VFILTER})"
  echo "    --cc           : Customer Code (3 lower-case letters)"
  echo "    --all          : do ALL customer codes"
  echo "    -e             : Etc/clusters file (${CLUSTERS})"
  echo "    -h | --help    : this help"
  echo "    -m | --mail    : do send Mail"
  echo "    --mailto       : change MAILTO address & do send mail (${MAILTO})"
  echo "    -q             : Qtree reporting"
  echo "    -s             : Snapshot reporting"
  echo "    -V             : Version"
  echo "    -x             : set -x"
}

# Check options
while [ ${#} -gt 0 ]
  do
  case ${1} in
    --all) DOALL=1  ;;
    -c | --cluster) CFILTER=${2}; shift ;;
    --cc) CUSTCODE=${2}; shift ;;
    -e) CLUSTERS=${2}; shift ;;
    -f) FILTER=${2}; shift ;;
    -v) VFILTER=${2}; shift ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    --mailto) MAILTO=${2}; MAIL=1; shift ;;
    -q) QTREERPT=1 ;;
    -s) SNAP_RPT=1 ;;
    --svm) SFILTER=${2}; CUSTCODE=${SFILTER}; shift ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done

# Warning for no -m | --mail
if [ ! ${MAIL} ]; then
  echo ""
  echo "  WARNING: Mailing (-m | --mail) NOT selected. If OK, continue. Otherwise abort by ^C"|tee -a ${LOG}
  sleep 5
fi  # ! MAIL

#if [ ! ${DOALL} ] && [ -z "${CUSTCODE}" ]; then
#  echo "`date` ${PGM}: ERROR; No (part of) customer code given. So not to know what to do. Exiting..."|tee -a ${LOG} ${ERR} ; echo; USAGE;
#  exit 1
#fi

if [ ${DOALL} ]; then
  CUSTCODE="nlnaf"
#  VFILTER="vol"
else
  CUSTOMER=`echo "${CUSTCODE}"|tr [:lower:] [:upper:]`
fi


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}


#if [ -z "${CUSTCODE}" ]
#then
#  echo "`date` ${PGM}: ERROR; No (part of) customer code given. So not to know what to do. Exiting..."|tee -a ${LOG} ${ERR} ; echo; USAGE;
#  exit 1
#fi
if [ -z "${CUSTCODE}" ]; then
  CUSTCODE="nl"
fi

CUSTOMER=`echo "${CUSTCODE}"|tr [:lower:] [:upper:]`
CSV="FSOD-cDOT_Autosize_report_${CUSTOMER}_${DATE}.csv"
SNAPCSV="FSOD-cDOT_Snapshot_report_${CUSTOMER}_${DATE}.csv"
if [ "${CFILTER}" != '[?]*' ]; then
  CSV="FSOD-cDOT_Autosize_report_${CUSTOMER}-${CFILTER}_${DATE}.csv"
  SNAPCSV="FSOD-cDOT_Snapshot_report_${CUSTOMER}-${CFILTER}_${DATE}.csv"
fi

# VFILTER need to have something. So if empty, then will be CUSTCODE
#if [ "${VFILTER}" = "" ]; then
#  VFILTER=${CUSTCODE}
#fi


# MAIN
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG} ${ERR}
echo "CFILER=${CFILTER}"|tee -a ${MAILMSG}
echo "SFILTER=${SFILTER}"|tee -a ${MAILMSG}
echo "VFILTER=${VFILTER}"|tee -a ${MAILMSG}
echo "CSV=${CSV}"|tee -a ${MAILMSG}
echo "CUSTOMER=${CUSTOMER}"|tee -a ${MAILMSG}
echo "CUSTCODE=${CUSTCODE}"|tee -a ${MAILMSG}
echo "DOALL=${DOALL}"|tee -a ${MAILMSG}
echo "MAIL=${MAIL}"|tee -a ${MAILMSG}
echo "MAILTO=${MAILTO}"|tee -a ${MAILMSG}
echo "PREFIX=${PREFIX}"|tee -a ${MAILMSG}
echo "QTREERPT=${QTREERPT}"|tee -a ${MAILMSG}
echo "SNAPCSV=${SNAPCSV}"|tee -a ${MAILMSG}
echo "SNAP_RPT=${SNAP_RPT}"|tee -a ${MAILMSG}
sleep 1

touch ${TMP} ${TMP}.TOTALLIMIT ${TMP}.TOTALSNAPSIZE

echo "# CLUSTER;VSERVER;VOLUME;AGGREGATE;SIZE [GB];USED [GB];PERCENT-USED [%];max-autosize [GB];min-autosize [GB];autosize-grow-threshold-percent [%];autosize-shrink-threshold-percent [%];autosize-mode;TYPE;QTREE;disk-used [GB];QUOTA (disk-limit) [GB];DISKUSAGEPERC [%];TOTALLIMIT [GB];snapshot [GB];create-time;size [GB];usedblocks [%];TOTALSNAPUSAGE [GB];" > ${TMP}
echo "# CLUSTER;VSERVER;VOLUME;AGGREGATE;snapshot [GB];create-time;size [GB];usedblocks [%];TOTALSNAPUSAGE [GB];" > ${TMP}.snaprpt

for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep ${CFILTER}`
do
  echo "  ${CLUSTER}:" 
# option -field selects the items to be shown. NOT the order. That is FIXED

  SSHCMD ${CLUSTER} "set -showseparator \" \" ;vserver show -operational-state running -field vserver"|grep "${CUSTCODE}"|grep "${SFILTER}"|while read VSERVER REST
  do
    echo "  ${CLUSTER}:${VSERVER}:"


#  SSHCMD ${CLUSTER} "volume show -vserver nlnafs${CUSTCODE}* -state online -field volume"|grep ${PREFIX}|grep ${VFILTER}|awk '{print $2}'|while read VOLUME REST

    SSHCMD ${CLUSTER} "set -showseparator \" \" ;volume show -vserver ${VSERVER} -state online -field volume"|grep ${VSERVER}|grep "${VFILTER}"|awk '{print $2}'|while read VOLUME REST
    do
      echo "  ${CLUSTER}:${VSERVER}:${VOLUME} ..."

      echo -n "${CLUSTER};" >> ${TMP}
      echo -n "${CLUSTER};" >> ${TMP}.snaprpt

      SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ; volume show -vserver ${VSERVER} -volume ${VOLUME} -field aggregate,size,used,percent-used,autosize-grow-threshold-percent,autosize-shrink-threshold-percent,autosize-mode,min-autosize,max-autosize,type"|grep ${VSERVER}|sed 's/GB//g'|tee -a ${TMP}

      if [ ${QTREERPT} ]; then
# Get Quote info /qtree
        TOTALLIMIT=0
        SSHCMD ${CLUSTER} "qtree show -vserver ${VSERVER} -volume ${VOLUME} -fields qtree"|grep ${VSERVER}|awk '{print $3}'|while read QTREE REST
        do
          if [ "${QTREE}" != "\"\"" ]; then
            DISKUSED="`SSHCMD ${CLUSTER} "set -units GB ; quota report -vserver ${VSERVER} -volume ${VOLUME} -tree ${QTREE} -field vserver,volume,tree,disk-used"|grep ${VSERVER}|awk '{print $5}'|sed 's/GB//g'`"
            if [ "${DISKUSED}" = "" ]; then
              DISKUSED=0
            fi
            QUOTA="`SSHCMD ${CLUSTER} "set -units GB ; quota report -vserver ${VSERVER} -volume ${VOLUME} -tree ${QTREE} -field vserver,volume,tree,disk-limit"|grep ${VSERVER}|awk '{print $5}'|sed 's/GB//g'`"
            if [ "${QUOTA}" = "" ]; then
              QUOTA=0
            fi
            if [ ${QUOTA} -gt 0 ]; then
              TOTALLIMIT=`expr ${TOTALLIMIT} + ${QUOTA}`
            fi
# Calculate usage% (DISKUSED/QUOTA)
            DISKUSAGEPERC=""
            if [ ${DISKUSED} -gt 0 ] && [ ${QUOTA} -gt 0 ]; then
##echo "DISKUSED=${DISKUSED}| QUOTA=${QUOTA}|"
              DISKUSAGEPERC=`expr ${DISKUSED} \* 100 / ${QUOTA} \* 100 / 100`
            fi

            echo "${CLUSTER};${VSERVER};${VOLUME};;;;;;;;;;;${QTREE};${DISKUSED};${QUOTA};${DISKUSAGEPERC};;"|tee -a ${TMP}
          fi  # ${QTREE}" ! = ""
# To get the value out site the subshell-loop
          echo ${TOTALLIMIT} > ${TMP}.TOTALLIMIT
        done  # Qtree
        TOTALLIMIT=`cat ${TMP}.TOTALLIMIT`
        echo "${CLUSTER};${VSERVER};${VOLUME};;;;;;;;;;;;;;;${TOTALLIMIT};"|tee -a ${TMP}
      fi  # QTREERPT


      if [ ${SNAP_RPT} ]; then
# Reporting the snapshots, 
# Count all snapshots sizes together
        TOTALSNAPSIZE=0
        SSHCMD ${CLUSTER} "set -unit GB -showseparator \";\" ; snapshot show -vserver ${VSERVER} -volume ${VOLUME} -field snapshot,size,create-time,used"|grep ${VSERVER}|while read LINE
        do
          SNAPSHOT=`echo ${LINE}|awk -F\; '{print $3}'`
          CREATE_TIME=`echo ${LINE}|awk -F\; '{print $4}'`
          SIZE=`echo ${LINE}|awk -F\; '{print $5}'|sed 's/GB//g'`
          USEDBLOCKS=`echo ${LINE}|awk -F\; '{print $6}'`
          TOTALSNAPSIZE=`expr ${TOTALSNAPSIZE} + ${SIZE}`
          echo "${CLUSTER};${VSERVER};${VOLUME};;;;;;;;;;;;;;;;${SNAPSHOT};${CREATE_TIME};${SIZE};${USEDBLOCKS};;"|tee -a ${TMP}
          echo "${CLUSTER};${VSERVER};${VOLUME};${SNAPSHOT};${CREATE_TIME};${SIZE};${USEDBLOCKS};;"|tee -a ${TMP}.snaprpt
          echo ${TOTALSNAPSIZE} > ${TMP}.TOTALSNAPSIZE

        done  # while read LINE
        TOTALSNAPSIZE=`cat ${TMP}.TOTALSNAPSIZE`
        echo "${CLUSTER};${VSERVER};${VOLUME};;;;;;;;;;;;;;;;${SNAPSHOT};;;;${TOTALSNAPSIZE};"|tee -a ${TMP}
        echo "${CLUSTER};${VSERVER};${VOLUME};${SNAPSHOT};;;;${TOTALSNAPSIZE};"|tee -a ${TMP}.snaprpt
      else
# If no SNAP_RPT then "close" (of end) report-line
       echo ""|tee -a ${TMP}.snaprpt
      fi  # SNAP_RPT

    done  # VOLUME

  done  # VSERVER
done  # for CLUSTER

if [ ${MAIL} ]; then
  echo "#" >> ${TMP}
  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" >> ${TMP}
  cp ${TMP} /tmp/${CSV}
  date | tee -a ${MAILMSG}
  cat ${MAILMSG} | mailx -a /tmp/${CSV} -s "@${HOSTNAME}: Autosize report from ${DATE} (for ${CUSTCODE}) [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed autosize-report to ${MAILTO}"|tee -a ${LOG}
  if [ ${SNAP_RPT} ]; then
    cp ${TMP}.snaprpt /tmp/${SNAPCSV}
    cat ${MAILMSG} | mailx -a /tmp/${SNAPCSV} -s "@${HOSTNAME}: Snapshot report from ${DATE} (for ${CUSTCODE}) [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed snapshot-report to ${MAILTO}"|tee -a ${LOG}
    rm /tmp/${SNAPCSV}
  fi
  rm /tmp/${CSV}

fi  #  MAIL

# Finish
rm ${TMP} ${TMP}.snaprpt ${TMP}.TOTALLIMIT ${TMP}.TOTALSNAPSIZE ${MAILMSG}
echo "`date` ${PGM} finished."|tee -a ${LOG}
exit 0