#!/bin/ksh
# File  : cdot_chk-rpt_qos_settings.sh
# By    : Maarten de Boer, 180417
# Subject       : Script to report (in Graphite) QoS
# set -x
#(0.2),180417	: Added -g & -m
#(0.3),200310	: Add filter for SVM (SFILTER). Add AGGR. MAX_QOS with PCT
#(0.4),200318	: Add; --noerror, for not sending error mail
#(0.5),200422	: Add; report on "wrong" aggr configured (AGGRCHK)
#               : Mod; -admin-state running,stopped
#(0.6),200429   : Mod; FILTER -> CFILER (-c|--cluster)
#(0.7),200429	: Mod; NO noerrmsg & No Single ErrMsg by mail
#(0.8),200429	: Mod; No ERROR at CC-clusters (EXCLUDES) 
#(0.9),200430	: Mod; "atos.qos...max_ops
#(0.10),200506	: Mod; "atos.qos...total_size
#(0.11),200519	: Some mod's at mailx, Added COMMENT
#(0.12).200526	: Split report of Wrong storage-class & No QoS-policy, add CREATE_TIME & COMMENT in CSV
#(0.13),200615	: Add QoS Adaptive Policy Group Name
#(0.14),200617	: Add COMMENT & CREATE_TIME at "Wrong storage-class"
#(0.15),210825	: Add --add-excludes & DATI
#(0.16),211110	: When Q-QOS used in rpt mode = OK, no ERROR
#(0.17),220413	: Mod. Check nc only if GRAPHITE & NC
#(0.18),220727	: Mod. AGGRBRONZE for n14a9001 
#(0.19),220803	: Mod. AGGR's
#(0.20),220815	: Mod. SSHCMD & Chk/Rpt vol0root as well. "vol0root" might not have QoS. So, exclude (egrep -v 'root|vol0ls') of the past was OK.
#(0.21),220818	: Mod AGGRSILVER
#(0.22),220831	: Mod. echo's in mail, Added VERBOSE -v, Mod. TOTAL_SIZE_GB
#(0.23),220831	: Renamed from cdot_rpt_qos_settings.sh
#(0.24),220921	: Added DISCLAIMER
#(0.25),220927	: Mod. EXCLUDES
PGM="`basename $0|cut -d\. -f1`"
VER="0.25"
TMP="/tmp/${PGM}.$$"
CSV="${PGM}.csv"
CLUSTERS="${HOME}/etc/clusters"
SSH="/usr/bin/ssh"
HOSTNAME="`hostname | cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
ASC="${PGM}.asc"
CFILTER="[?]*"
MAIL=""
MAILTO="maarten.deboer@atos.net"
NC="${HOME}/bin/nc"
GRAPHITESRV="10.192.109.224 2003"
GRAPHITE=""
CSV="/tmp/${PGM}.csv"
SFILTER="[?]*"
ERRFILE=${TMP}.errfile
EXCLUDES=${HOME}/etc/${PGM}.excl
COMMENT=""
NOQOSPOL=""
WRONGSC=""
CREATE_TIME=""
DATI=`date +%Y-%m-%d-%H-%M-%S`
VERBOSE=""
DISCLAIMER="${HOME}/etc/${PGM}.disclaimer"

AGGRCHK=""
# Diamond = AFF: n??a3801
AGGRDIAMOND="^n[0-9][0-9]a[3][8][0-9][0-9]"
# Platinum = AFF: n??a3801
AGGRPLATINUM="^n[0-9][0-9]a[3][8][0-9][0-9]"
# Gold = AFF: n??a3801, SAS: n??a0601, n??a0901, n??a1201
AGGRGOLD="^n[0-9][0-9]a[013][6928][0-9][0-9]"
# Silver = AFF: n??a3801, SATA: n??a2001
#AGGRSILVER="n[0-9][0-9]a20[0-9][0-9]"
#AGGRSILVER="^n[0-9][0-9]a[23][08][0-9][0-9]"
#add GOLD aggr's (a1201) into it as well
AGGRSILVER="^n[0-9][0-9]a[123][028][0-9][0-9]"
# Bronze = AFF: n??a3801, SATA: n??a3001,n??a4001,n??a7001,n??a8001,n??a9001
#AGGRBRONZE="^n[0-9][0-9][ace][3489]0[0-9][0-9]"
AGGRBRONZE="^n[0-9][0-9][ace][34789][08][0-9][0-9]"


# PREFIX needed at 'vserver show'
PREFIX="nl"

# QoS -max-throughput
# Storage classes
# Diamond  |               : 5000 IOPS /TB (1-2ms)   | +200%
# Platinum | First (FC)    : 1500 IOPS /TB (3-5 ms)  | +200%
# Gold     | Business (BC) :  500 IOPS /TB (4-8 ms)  | +200%
# Silver   | Economy (EC)  :  250 IOPS /TB (6-12 ms) | +150%
# Bronze   | Archive (AC)  :   25 IOPS /TB (8-28 ms) | +100%
# MAX_PT = Max percentage as MAX QoS
STORAGECLASSES="diamond platinum gold silver bronze"
DIAMONT_TB=5000
DIAMONT_MAX_PT=120
PLATINUM_TB=1500
PLATINUM_MAX_PT=200
GOLD_TB=500
#GOLD_MAX_PT=150
GOLD_MAX_PT=200
SILVER_TB=250
SILVER_MAX_PT=150
BRONZE_TB=25
BRONZE_MAX_PT=200


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -05 - -10 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -10
          fi  # -09
        fi  # -08
      fi  # -07
    fi  # -06
  fi  # -05
  rm ${TMPERR}
}


USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options          :"
  echo "    -a|--aggrchk   : Aggr check"
  echo "    -c|--cluster   : filter for clustername (${CFILTER})"
  echo "    -e|--etc       : Etc/clusters-file (${CLUSTERS})"
  echo "    -g|--graphite  : send to Graphite"
  echo "    -h|--help      : this Help"
  echo "    -m|--mail      : do send Mail"
  echo "    -s|--svm       : filter for SVM (${SFILTER})"
  echo "    -v|--verbose   : enable Verbose (echo more info)"
  echo "    -w|--wrong     : check & report Wrong storage-classes"
  echo "    -V             : show Version"
  echo "    -x             : set -x"
  echo "    --add-excludes : adding excludes to exclude file"
  echo "    --mailto       : change MAILTO address & do send mail (${MAILTO})"
  echo "    --noerrmsg     : send NO Error-Msg (to ${MAILTO})"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -a | --aggrchk) AGGRCHK=1 ;;
    -e | --etc) CLUSTERS="${2}"; shift ;;
    -c | --cluster) CFILTER="${2}"; shift ;;
    -s | --svm) SFILTER="${2}"; shift ;;
    --mailto) MAIL=1; MAILTO="${2}"; shift ;;
    -m | --mail) MAIL=1 ;;
    -v | --verbose) VERBOSE=1 ;;
    -w | --wrong) WRONGSC=1 ;;
    -g | --graphite) GRAPHITE=1 ;;
    --add-excludes) ADD_EXCLUDES=1; WRONGSC=1 ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;;
  esac
    shift
done  # case

# MAIN
echo "`date` ${PGM} v${VER} started"|tee -a ${LOG}
if [ ${GRAPHITE} ] && [ ! -f ${NC} ]; then
  echo "  nc ($NC} NOT found. Exiting ..."|tee -a ${LOG}
  exit 4
fi


echo "`date` ${PGM} v${VER}" > ${TMP}.mail
echo "" >> ${TMP}.mail
if [ -f ${DISCLAIMER} ]; then
  cat ${DISCLAIMER} >> ${TMP}.mail
fi  # DISCLAIMER
echo "" >> ${TMP}.mail
echo "" >> ${TMP}.mail
echo "" >> ${TMP}.mail
echo "AGGRCHK=${AGGRCHK}"|tee -a ${TMP}.mail
echo "ADD_EXCLUDES=${ADD_EXCLUDES}"|tee -a ${TMP}.mail
echo "CFILTER=${CFILTER}"|tee -a ${TMP}.mail
echo "CLUSTERS=${CLUSTERS}"|tee -a ${TMP}.mail
echo "DATI=${DATI}"|tee -a ${TMP}.mail
echo "ERRFILE=${ERRFILE}"|tee -a ${TMP}.mail
echo "EXCLUDES=${EXCLUDES}"|tee -a ${TMP}.mail
echo "GRAPHITE=${GRAPHITE}"|tee -a ${TMP}.mail
echo "GRAPHITESRV=${GRAPHITESRV}"|tee -a ${TMP}.mail
echo "MAIL=${MAIL}"|tee -a ${TMP}.mail
echo "MAILTO=${MAILTO}"|tee -a ${TMP}.mail
echo "SFILTER=${SFILTER}"|tee -a ${TMP}.mail
echo "VERBOSE=${WRONGSC}"|tee -a ${TMP}.mail
echo "WRONGSC=${WRONGSC}"|tee -a ${TMP}.mail
echo "DISCLAIMER=${DISCLAIMER}"|tee -a ${TMP}.mail

if [ ${WRONGSC} ] && [ -f ${EXCLUDES} ]; then
  echo "  check WrongStorageClass & EXCLUDES (${EXCLUDES}) found." |tee -a ${LOG}
  if  [ ${VERBOSE} ]; then
    cat ${EXCLUDES}|grep -v ^#
  fi  # VERBOSE
  sleep 1
fi
sleep 1

touch ${TMP} 
if [ ${WRONGSC} ]; then
  date > ${ERRFILE}
  echo "    ERROR-file (${ERRFILE}) attached in the mail" >> ${TMP}.mail
fi  # WRONGSC

if [ ${ADD_EXCLUDES} ]; then
# Save "old" file 1st
  cp ${EXCLUDES} ${EXCLUDES}_${DATI}
  echo "# (`date`) Adding excludes (by option --add-excludes)" >> ${EXCLUDES}
fi

cat "${CLUSTERS}"|grep -v \^#|awk -F\; '{print $1}'|sort|grep "${CFILTER}"|while read CLUSTER
do
  echo "  ${CLUSTER} ..."

# Getting the nodes 1st
# Also the DR (stopped) ones
  SSHCMD ${CLUSTER} "vserver show -admin-state running,stopped"|grep ${PREFIX}|grep ${SFILTER}|awk '{print $1}'|while read VSERVER
  do
    echo "  ${CLUSTER};${VSERVER} ..."

# Exclude root- & LS-volumes
    SSHCMD ${CLUSTER} "set -units GB -showseparator \";\"; vol show -vserver ${VSERVER} -field  aggregate,qos-policy-group,qos-adaptive-policy-group,state,total,comment,create-time"|grep ${VSERVER}|egrep -v 'root|vol0ls'|while read LINE
    do
#Vserver Name;Volume Name;Aggregate Name;Volume State;Comment;Total User-Visible Size;Creation Time;QoS Policy Group Name;QoS Adaptive Policy Group Name;
      SVM=`echo ${LINE}|awk -F\; '{print $1}'`
      VOL=`echo ${LINE}|awk -F\; '{print $2}'`
      AGGR=`echo ${LINE}|awk -F\; '{print $3}'`
      VOL_STATE=`echo ${LINE}|awk -F\; '{print $4}'`
      COMMENT=`echo ${LINE}|awk -F\; '{print $5}'`
      TOTAL_SIZE_GB=`echo ${LINE}|awk -F\; '{print $6}'|sed 's/GB//g'`
      CREATE_TIME=`echo ${LINE}|awk -F\; '{print $7}'`
      QOS_POL_GR_NAME=`echo ${LINE}|awk -F\; '{print $8}'`
      QOS_A_POL_GR_NAME=`echo ${LINE}|awk -F\; '{print $9}'`

      if [ "${TOTAL_SIZE_GB}" = "-" ]; then
        TOTAL_SIZE_GB=0
      fi

      STOR_CLASS="-"
# Stor class comes from Qos or A-qos
      if [ "${QOS_POL_GR_NAME}" != "-" ]; then
        STOR_CLASS=`echo ${QOS_POL_GR_NAME}|awk -F\_ '{print $2}'`
      fi
      if [ "${QOS_A_POL_GR_NAME}" != "-" ]; then
        STOR_CLASS=`echo ${QOS_A_POL_GR_NAME}|awk -F\_ '{print $2}'`
      fi

      touch ${TMP} 

# Check "right" AGGR
# MAX_QOS = needed to graphite-grafana "insertion". For reporting on MAX-QoS
      case ${STOR_CLASS} in
        bronze)
          let MAX_QOS=${TOTAL_SIZE_GB}*${BRONZE_TB}*${BRONZE_MAX_PT}/100/1024
          if [ ${MAX_QOS} -lt ${BRONZE_TB} ]; then
            let MAX_QOS=${BRONZE_TB}*${BRONZE_MAX_PT}/100
          fi
          VOLISBRONZE=`echo "${AGGR}"|grep "${AGGRBRONZE}"`
          if [ ${WRONGSC} ] && [ "${VOLISBRONZE}" = "" ]; then
# Check if A-QOS is not set (= "-") . Then ERROR.
            if [ "${QOS_A_POL_GR_NAME}" = "-" ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE}
              else
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|tee -a ${ERRFILE}
              fi  # EXCLUDES
            fi  # QOS_A_POL_GR_NAME}" = "-" 
# Add option
# nlnaf101;nlnafsakz05;n03a0601;akz05_vol01084
            if [ ${ADD_EXCLUDES} ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "${CLUSTER};${SVM};${AGGR};${VOL}"|grep -v -f ${EXCLUDES}  >> ${EXCLUDES}
              else
                echo "${CLUSTER};${SVM};${AGGR};${VOL}" >> ${EXCLUDES}
              fi  # -f ${EXCLUDES
            fi  # ADD_EXCLUDES
          fi
          ;;
        silver) 
          let MAX_QOS=${TOTAL_SIZE_GB}*${SILVER_TB}*${SILVER_MAX_PT}/100/1024
          if [ ${MAX_QOS} -lt ${SILVER_TB} ]; then
            let MAX_QOS=${SILVER_TB}*${SILVER_MAX_PT}/100
          fi
          VOLISSILVER=`echo "${AGGR}"|grep "${AGGRSILVER}"`
          if [ ${WRONGSC} ] && [ "${VOLISSILVER}" = "" ]; then
# Check if A-QOS is not set (= "-") . Then ERROR.
            if [ "${QOS_A_POL_GR_NAME}" = "-" ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE}
              else
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|tee -a ${ERRFILE}
              fi  # -f ${EXCLUDES}
            fi  # QOS_A_POL_GR_NAME}" = "-" 

# Add option
# nlnaf101;nlnafsakz05;n03a0601;akz05_vol01084
            if [ ${ADD_EXCLUDES} ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "${CLUSTER};${SVM};${AGGR};${VOL}"|grep -v -f ${EXCLUDES}  >> ${EXCLUDES}
              else
                echo "${CLUSTER};${SVM};${AGGR};${VOL}" >> ${EXCLUDES}
              fi  # -f ${EXCLUDES
            fi  # ADD_EXCLUDES
          fi
          ;;
        gold) 
          let MAX_QOS=${TOTAL_SIZE_GB}*${GOLD_TB}*${GOLD_MAX_PT}/100/1024
          if [ ${MAX_QOS} -lt ${GOLD_TB} ]; then
            let MAX_QOS=${GOLD_TB}*${GOLD_MAX_PT}/100
          fi
          VOLISGOLD=`echo ${AGGR}|grep "${AGGRGOLD}"`
          if [ ${WRONGSC} ] && [ "${VOLISGOLD}" = "" ]; then
# Check if A-QOS is not set (= "-") . Then ERROR.
            if [ "${QOS_A_POL_GR_NAME}" = "-" ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE}
              else
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|tee -a ${ERRFILE}
              fi
            fi  # QOS_A_POL_GR_NAME}" = "-" 
# Add option
# nlnaf101;nlnafsakz05;n03a0601;akz05_vol01084
            if [ ${ADD_EXCLUDES} ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "${CLUSTER};${SVM};${AGGR};${VOL}"|grep -v -f ${EXCLUDES}  >> ${EXCLUDES}
              else
                echo "${CLUSTER};${SVM};${AGGR};${VOL}" >> ${EXCLUDES}
              fi  # -f ${EXCLUDES
            fi  # ADD_EXCLUDES
          fi
          ;;
        platinum) 
          let MAX_QOS=${TOTAL_SIZE_GB}*${PLATINUM_TB}*${PLATINUM_MAX_PT}/100/1024 
          if [ ${MAX_QOS} -lt ${PLATINUM_TB} ]; then
            let MAX_QOS=${PLATINUM_TB}*${PLATINUM_MAX_PT}
          fi
          VOLISPLATINUM=`echo "${AGGR}"|grep "${AGGRPLATINUM}"`
          if [ ${WRONGSC} ] && [ "${VOLISPLATINUM}" = "" ]; then
# Check if A-QOS is not set (= "-") . Then ERROR.
            if [ "${QOS_A_POL_GR_NAME}" = "-" ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE}
              else
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|tee -a ${ERRFILE}
              fi
            fi  # QOS_A_POL_GR_NAME}" = "-" 
# Add option
# nlnaf101;nlnafsakz05;n03a0601;akz05_vol01084
            if [ ${ADD_EXCLUDES} ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "${CLUSTER};${SVM};${AGGR};${VOL}"|grep -v -f ${EXCLUDES}  >> ${EXCLUDES}
              else
                echo "${CLUSTER};${SVM};${AGGR};${VOL}" >> ${EXCLUDES}
              fi  # -f ${EXCLUDES
            fi  # ADD_EXCLUDES
          fi
          ;;
        diamond) 
          let MAX_QOS=${TOTAL_SIZE_GB}*${DIAMONT_TB}*${DIAMONT_MAX_PT}/100/1024 
          if [ ${MAX_QOS} -lt ${DIAMONT_TB} ]; then
            let MAX_QOS=${DIAMONT_TB}*${DIAMONT_MAX_PT}
          fi
          VOLISDIAMOND=`echo "${AGGR}"|grep "${AGGRDIAMOND}"`
          if [ ${WRONGSC} ] && [ "${VOLISDIAMOND}" = "" ]; then
# Check if A-QOS is not set (= "-") . Then ERROR.
            if [ "${QOS_A_POL_GR_NAME}" = "-" ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE}
              else
                echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|tee -a ${ERRFILE}
              fi
            fi  # QOS_A_POL_GR_NAME}" = "-" 
# Add option
# nlnaf101;nlnafsakz05;n03a0601;akz05_vol01084
            if [ ${ADD_EXCLUDES} ]; then
              if [ -f ${EXCLUDES} ]; then
                echo "${CLUSTER};${SVM};${AGGR};${VOL}"|grep -v -f ${EXCLUDES}  >> ${EXCLUDES}
              else
                echo "${CLUSTER};${SVM};${AGGR};${VOL}" >> ${EXCLUDES}
              fi  # -f ${EXCLUDES
            fi  # ADD_EXCLUDES
          fi
          ;;
        *) 
          MAX_QOS=0 
          if [ ${WRONGSC} ]; then 
            if [ -f ${EXCLUDES} ]; then
              echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE}
            else
              echo "  ERROR: Wrong storage-class? QoS-policy=${STOR_CLASS} <> Aggr:${AGGR} of ${CLUSTER};${SVM};${AGGR};${VOL} (${TOTAL_SIZE_GB}GB,${COMMENT},${CREATE_TIME})"|tee -a ${ERRFILE}
            fi  # EXCLUDES
          fi  # WRONGSC
          ;;
      esac

      if [ ${VERBOSE} ]; then
        echo "${CLUSTER};${SVM};${AGGR};${VOL};${TOTAL_SIZE_GB};${QOS_POL_GR_NAME};${QOS_A_POL_GR_NAME};${STOR_CLASS};${MAX_QOS};${DATE_S};${COMMENT};${CREATE_TIME};"|tee -a ${TMP}
      else  # VERBOSE
        echo "${CLUSTER};${SVM};${AGGR};${VOL};${TOTAL_SIZE_GB};${QOS_POL_GR_NAME};${QOS_A_POL_GR_NAME};${STOR_CLASS};${MAX_QOS};${DATE_S};${COMMENT};${CREATE_TIME};" >> ${TMP}
      fi  # VERBOSE

# Check QoS-policy
# And send to Graphite
      if [ "${QOS_POL_GR_NAME}" != "-" ] || [ "${QOS_A_POL_GR_NAME}" != "-" ]; then
        if [ ${GRAPHITE} ]; then
          DATE_S="`date +%s`"
        else
          DATE_S=""
        fi
        if [ ${GRAPHITE} ]; then
          echo "atos.qos.nl.${CLUSTER}.svm.${SVM}.vol.${VOL}.total_size ${TOTAL_SIZE_GB} ${DATE_S}"|${NC} ${GRAPHITESRV}
          echo "atos.qos.nl.${CLUSTER}.svm.${SVM}.vol.${VOL}.max_ops ${MAX_QOS} ${DATE_S}"|${NC} ${GRAPHITESRV}
        fi  # GRAPHITE
      else
        if [ -f ${EXCLUDES} ]; then
          echo "  ! ${CLUSTER};${SVM};${AGGR};${VOL} has NO QoS-policy-group defined (${QOS_POL_GR_NAME}|${QOS_A_POL_GR_NAME}) [${COMMENT}] (${CREATE_TIME})"|grep -v -f ${EXCLUDES}|tee -a ${ERRFILE} ${LOG}
        else
          echo "  ! ${CLUSTER};${SVM};${AGGR};${VOL} has NO QoS-policy-group defined (${QOS_POL_GR_NAME}|${QOS_A_POL_GR_NAME}) [${COMMENT}] (${CREATE_TIME})"|tee -a ${ERRFILE} ${LOG}
        fi  # EXCLUDES

        sleep 1
      fi  # ! -

      STOR_CLASS=""

    done  # read LINE

  done  # "vserver show
# If ERRFILE is > 0, then add HR (--------------------)
  if [ -s ${ERRFILE} ]; then # If size > 0
    echo "--------------------------------------------------" >> ${ERRFILE}
  fi
done  # CLUSTER


if [ ${MAIL} ]; then
  echo "# CLUSTER;SVM;AGGR;VOL;TOTAL_SIZE_GB;QOS_POL_GR_NAME;QOS_A_POL_GR_NAME;STOR_CLASS;MAX_QOS;DATE_S (if empty not send to Graphite);COMMENT;CREATE_TIME;" > ${CSV}
  cat ${TMP} >> ${CSV}
# If ERRFILE, the add this to the mail. Otherwise only add the CSV to the mail
  if [ -s ${ERRFILE} ]; then # If size > 0
    cat ${TMP}.mail | mailx -a ${ERRFILE} -a ${CSV} -s ":${HOSTNAME}: ERROR & QoS (CSV) check report [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed ERROR (${ERRFILE}) & report (${CSV}) to ${MAILTO}"|tee -a ${LOG}
    rm ${ERRFILE}
  else
    cat ${TMP}.mail | mailx -a ${CSV} -s ":${HOSTNAME}: QoS (CSV) report [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed report (${CSV}) to ${MAILTO}"|tee -a ${LOG}
  fi  # ERRFILE
  rm ${CSV} 
fi  # MAIL

rm ${TMP} ${TMP}.mail
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0