#!/bin/ksh
# File  : cdot_set_qos_settings.sh
# By    : Maarten de Boer, 180116
# Subject       : Script to set QoS settings
# set -x
# ToDos:
# Cleaning up not used QoS-policies. But NOT the 0-INF ones
# Check if right policies defined (<SVM>_class_ ). 
# If not ERROR & mail
#(0.4),180118	: Mod incident when no qos-policy-group at a volume
#(0.5),180118	: Added     --setsize) SETQOS=1 ;;
#(0.6),180126	: Mod MAX_PT's
#(0.7),180126	: Add --setpolicy for setting the INF policy to the volumes
#(0.8),180126	: Added VFILTER
#(0.9),200310	: Mod to SFILTER (SVMs) (removed VPREFIX)
#(0.10),200319	: Add AGGRPLATINUM
#(0.11),200326	: Mod AGGRBRONZE
#(0.12),200331	: Added check if no qos-policy-group is onto volume (at SETPOLICY). Otherwise do not set.
#		  Change -f(ilter) into -c(luster) 
#                 vserver operational-state => admin-state running,stopped
#(0.13),200406	: Added VOLume Filter VFILTER
#(0.14),200611	: Remove Nagios, Add SHOWPOLICY
#(0.15),200611  : Add --setaqos
#(0.16),200615	: Mod; --showpolicy => showqos, NO WARNINGSTR mailed
#(0.17),200617	: Add; check StorageClass. Otherwise exit(5)
#
PGM="`basename $0|cut -d\. -f1`"
VER="0.17"
TMP="/tmp/${PGM}.$$"
CLUSTERS="${HOME}/etc/clusters"
MAILTO="maarten.deboer@atos.net"
SSH="/usr/bin/ssh"
HOSTNAME="`hostname | cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
CFILTER="[?]*"
MAIL=""
SETQOS=""
SETPOLICY=""
SFILTER="[?]*"
VFILTER="[?]*"
SHOWQOS=""
CSV="/tmp/${PGM}.csv"
SETAQOS=""
AQOS_CLASS=""
LISTQOS=""

# PREFIX needed at 'vserver show'
PREFIX="nl"

# QoS -max-throughput 
MAXQTP="INF"
# Storage classes
# Diamond  |               : 5000 IOPS /TB (1-2ms)
# Platinum | First (FC)    : 1500 IOPS /TB (3-5 ms)
# Gold     | Business (BC) :  500 IOPS /TB (4-8 ms)
# Silver   | Economy (EC)  :  250 IOPS /TB (6-12 ms)
# Bronze   | Archive (AC)  :   25 IOPS /TB (8-28 ms)
# MAX_PT = Max percentage as MAX QoS
STORAGECLASSES="diamond platinum gold silver bronze"
DIAMOND_MIN=1500
DIAMOND_TB=5000
DIAMOND_PEAK=6000
PLATINUM_MIN=450
PLATINUM_TB=1500
PLATINUM_PEAK=3000
GOLD_MIN=150
GOLD_TB=500
GOLD_PEAK=1000
SILVER_MIN=250
SILVER_TB=250
SILVER_PEAK=375
BRONZE_MIN=25
BRONZE_TB=25
BRONZE_PEAK=25

# AGGRlist /CLASS

#AGGRPLATINUM="n[0-9][0-9]a[3][8][0-9][0-9]"
#AGGRGOLD="n[0-9][0-9]a[01][962][0-9][0-9]"

# Gold is SAS and also SSD
AGGRGOLD="n[0-9][0-9]a[013][6928][0-9][0-9]"

AGGRSILVER="n[0-9][0-9]a20[0-9][0-9]"
AGGRBRONZE="n[0-9][0-9][ace][348]0[0-9][0-9]"


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}


USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options        :"
  echo "    -h|--help    : this Help"
  echo "    -l|--list    : List qos"
  echo "    -c|--cluster : Cluster filter (${CFILTER})"
  echo "    -s|--svm     : SVM filter (${SFILTER})"
  echo "    -v|--vol     : VOLume filter (${VFILTER})"
  echo "    -e|--etc     : Etc/clusters-file (${CLUSTERS})"
  echo "    -m|--mail    : do send Mail"
  echo "    -V           : show Version"
  echo "    -x           : set -x"
  echo "    --mailto     : change MAILTO address & do send mail (${MAILTO})"
#  echo "    --setqos     : do SETQOS"
  echo "    --setaqos    : setaqos <aqos_class> "
  echo "    --setpolicy  : Set the INF (max) policy to the volumes"
  echo "    --showqos    : Show the qos of the volumes"
}
## MAIN
# Check options
if [ $# -eq 0 ]; then
  echo "No option(s) given. So not to know what to do. Exiting..."; echo; USAGE;
 exit 1
fi
while [ $# -gt 0 ]
  do
  case $1 in
    -e | --etc) CLUSTERS=$2; shift ;;
    -c | --cluster) CFILTER=$2; shift ;;
    -s | --svm) SFILTER=$2; shift ;;
    -v | --vol) VFILTER=$2; shift ;;
    -l | --list) LISTQOS=1 ;;
    -m | --mail) MAIL=1 ;;
    --mailto) MAILTO=$2; MAIL=1; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    --setaqos) SETAQOS=1; AQOS_CLASS=$2; shift ;;
    --setpolicy) SETPOLICY=1 ;;
    --showqos) SHOWQOS=1 ;;
    *)  echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;;
  esac
    shift
done  # case

# MAIN

echo "`date` ${PGM} v${VER} started (CLUSTERS=${CLUSTERS},CFILTER=${CFILTER},MAIL=${MAIL},SETQOS=${SETQOS},SETAQOS=${SETAQOS},AQOS_CLASS=${AQOS_CLASS},SETPOLICY=${SETPOLICY},SFILTER=${SFILTER},SHOWQOS=${SHOWQOS},VFILTER=${VFILTER})"|tee -a ${LOG}
sleep 1

if [ ${SETAQOS} ]; then
  if [ "${VFILTER}" = "[?]*" ]; then
    echo ""
    echo "  For setting AQoS NO Volume-filter (${VFILTER}) is set. Exiting ..."|tee -a ${LOG}
    exit 4
  fi  # VFILTER
# Check is storageClass is a right one. Otherwise exit(5)
  SCLASS=`echo ${STORAGECLASSES}|grep ${AQOS_CLASS}`
  if [ "${SCLASS}" = "" ]; then
    echo "  NOT a right (${AQOS_CLASS}) storageclass ( => ${STORAGECLASSES}). Exiting ..."|tee -a ${LOG}
    exit 5
  fi  # SCLASS
fi

touch ${TMP}
echo "# Vserver Name;Volume Name;Aggregate Name;Comment;Total User-Visible Size;Volume Type;Create Time;QoS Policy Group Name;QoS Adaptive Policy Group Name" > ${TMP}

cat "${CLUSTERS}"|grep -v \^#|awk -F\; '{print $1}'|sort|grep "${CFILTER}"|while read CLUSTER
do
#  echo "  ${CLUSTER} ..."
# Getting the nodes 1st
  SSHCMD ${CLUSTER} "vserver show -admin-state running,stopped"|grep ${PREFIX}|grep ${SFILTER}|awk '{print $1}' > ${TMP}.2
  cat ${TMP}.2|while read VSERVER
  do
# The 1st line (without LF) can be :
#   ssh: connect to host nlnaf100-01 port 22: Connection refused
# The 2nd line can be:
#   nlnaf100-04 205 days 02:54
# So need the check this out
#echo "1st SSHCMD:"
    echo "  ${CLUSTER}/${VSERVER}..."

# Check if policy-groups /SVM are created by --setpolicy
    if [ ${SETPOLICY} ]; then
      for CLASS in ${STORAGECLASSES}
      do
        MAXTHP=`SSHCMD ${CLUSTER} "qos policy-group show -vserver ${VSERVER} -policy-group ${VSERVER}_${CLASS} -field max-throughput"|grep ${VSERVER}|awk '{print $2}'`
        echo "    policy-group:${VSERVER}_${CLASS}=${MAXTHP}"
        if [ "${MAXTHP}" !=  "INF" ]; then
          echo "  NO policy-group set for ${VSERVER} ${CLASS}"|tee -a ${LOG}
          echo "  SSHCMD ${CLUSTER} qos policy-group create -vserver ${VSERVER} -policy-group ${VSERVER}_${CLASS} -max-throughput INF"|tee -a ${LOG}
          SSHCMD ${CLUSTER} "qos policy-group create -vserver ${VSERVER} -policy-group ${VSERVER}_${CLASS} -max-throughput INF"
        fi  # MAXTHP
      done  # CLASS
    fi  # SETPOLICY

# Per volume /SVM
# Leave out SVM-root-vols (with "root" in the volume name)
# v11: Added root-vols to
# v12: Exclude LS-volume, so type added
    SSHCMD ${CLUSTER} "set -units GB -showseparator \";\"; volume show -vserver ${VSERVER} -field vserver,volume,aggregate,total,type,qos-policy-group,qos-adaptive-policy-group,comment,create-time"|grep ${PREFIX}|grep "${VFILTER}"|while read LINE
    do
#      echo ${LINE}
      VOL=`echo ${LINE}|awk -F\; '{print $2}'`
      AGGR=`echo ${LINE}|awk -F\; '{print $3}'`
      COMMENT=`echo ${LINE}|awk -F\; '{print $4}'`
      TOTAL=`echo ${LINE}|awk -F\; '{print $5}'`
      VOLTYPE=`echo ${LINE}|awk -F\; '{print $6}'`
      CREATE_TIME=`echo ${LINE}|awk -F\; '{print $7}'`
      QOSPOLGR=`echo ${LINE}|awk -F\; '{print $8}'`
      AQOSPOLGR=`echo ${LINE}|awk -F\; '{print $9}'`

      if [ "${QOSPOLGR}" != "-" ]; then
        QOS_CLASS=`echo ${QOSPOLGR}|cut -d\_ -f2`
      fi 
      if [ "${AQOSPOLGR}" != "-" ]; then
        QOS_CLASS=`echo ${AQOSPOLGR}|cut -d\_ -f2`
      fi 


      echo "  ${CLUSTER}/${VSERVER}:${AGGR}/${VOL}(${VOLTYPE}=${TOTAL})=${QOSPOLGR}|${AQOSPOLGR}(${QOS_CLASS})"
 
      if [ "${QOSPOLGR}" != "-" ] && [ "${AQOSPOLGR}" != "-" ]; then
# If NO QoS set AND NO Q-QoS, then find out class 1st
        SCLASS=""
        VOLISBRONZE=`echo "${AGGR}"|grep "${AGGRBRONZE}"`
        VOLISSILVER=`echo "${AGGR}"|grep "${AGGRSILVER}"`
        VOLISGOLD=`echo "${AGGR}"|grep "${AGGRGOLD}"`
# No Platinum of Diamond Stor-Class Aggr's defined
#        VOLISPLATINUM=`echo "${AGGR}"|grep "${AGGRPLATINUM}"`
#        VOLISDIAMOND=`echo "${AGGR}"|grep "${AGGRDIAMOND}"`
        if [ "${VOLISBRONZE}" != "" ]; then
          SCLASS="bronze"
        fi
        if [ "${VOLISSILVER}" != "" ]; then
          SCLASS="silver"
        fi
        if [ "${VOLISGOLD}" != "" ]; then
          SCLASS="gold"
        fi
#        if [ "${VOLISPLATINUM}" != "" ]; then
#          SCLASS="platinum"
#        fi
#        if [ "${VOLISDIAMOND}" != "" ]; then
#          SCLASS="diamond"
#        fi

# ( qos-policy-group is empty ("-") )

# NO QoS-policy set. 
        WARNINGSTR="  NO qos-policy-group was set at volume ${CLUSTER}/${VSERVER}:${VOL}" 
        echo "  ${WARNINGSTR}"|tee -a ${LOG}

# When SCLASS is empty, ERROR. Else set QoS-policy
        if [ "${SCLASS}" = "" ];  then
          echo "  No StorageClass (${SCLASS}) found for ${CLUSTER}/${VSERVER}:${AGGR}/${VOL}"|tee -a ${LOG}

        else
# If --setpolicy, then set a INF policy onto a volume according the aggr.
# And No  LoadShare volume
          if [ ${SETPOLICY} ] && [ "${VOLTYPE}" != "LS" ]; then
            echo "  SSHCMD ${CLUSTER} volume modify -vserver ${VSERVER} -volume ${VOL} -qos-policy-group ${VSERVER}_${SCLASS}"|tee -a ${LOG}
            SSHCMD ${CLUSTER} "volume modify -vserver ${VSERVER} -volume ${VOL} -qos-policy-group ${VSERVER}_${SCLASS}"
          fi  # SETPOLICY
        fi  # SCLASS

#        if [ ${MAIL} ]; then
#          echo "${WARNINGSTR}"|mailx -s "No QoS-policy set [${PGM} v${VER}]" ${MAILTO}
#          echo "  ${WARNINGSTR}. Mailed to (No QoS-policy set [${PGM} v${VER}])${MAILTO}"|tee -a ${LOG}
#        fi  # MAIL

      fi  # ${QOSPOLGR}" = "-"

# Setting A(daptive)-QoS
# Check if volume has a QoS-policy set
#   If not, then this need to be set first. By another "round"
      if [ ${SETAQOS} ]; then
        echo "    Setting A(daptive) QoS"
        if [ "${AQOS_CLASS}" = "" ]; then
          echo "     NO a(daptive) QoS class (${AQOS_CLASS}) defined."|tee -a ${LOG}
        else
          NEW_QOSPOLGR="${VSERVER}_${AQOS_CLASS}_aqos"
          QOSPOLGR=`SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ;qos adaptive-policy-group show -vserver ${VSERVER} -policy-group ${NEW_QOSPOLGR} -field policy-group"|grep ${VSERVER}|awk -F\; '{print $1}'`
          if [ "${QOSPOLGR}" = "${NEW_QOSPOLGR}" ]; then
            echo "    Setting (A-QoS) ${NEW_QOSPOLGR} to ${VSERVER}:${VOL}"|tee -a ${LOG}
            echo "    SSHCMD ${CLUSTER} volume modify -vserver ${VSERVER} -volume ${VOL} -qos-adaptive-policy-group ${NEW_QOSPOLGR}"|tee -a ${LOG}
            SSHCMD ${CLUSTER} "volume modify -vserver ${VSERVER} -volume ${VOL} -qos-adaptive-policy-group ${NEW_QOSPOLGR}"

# Show & save for mail
            echo -n "    A-QoS set: " |tee -a ${LOG}
            SSHCMD ${CLUSTER} "set -units GB -showseparator \";\" ;volume show -vserver ${VSERVER} -volume ${VOL} -field qos-adaptive-policy-group,comment "|grep ${VSERVER}|tee -a ${LOG}
            SSHCMD ${CLUSTER} "set -units GB -showseparator \";\"; volume show -vserver ${VSERVER} -volume ${VOL} -field vserver,volume,aggregate,total,type,qos-policy-group,qos-adaptive-policy-group,comment,create-time"|grep "${VSERVER}" >> ${TMP}
            
          else
            echo "    New AQoS-policy (${AQOS_CLASS}) is not found at SVM (${VSERVER}) (${QOSPOLGR}<>${NEW_QOSPOLGR}). Creating it"|tee -a ${LOG}

            case ${AQOS_CLASS} in
              diamond) 
                NEW_MIN=${DIAMOND_MIN}
                NEW_EXPECTED=${DIAMOND_TB}
                NEW_PEAK=${DIAMOND_PEAK}
                ;;
              platinum) 
                NEW_MIN=${PLATINUM_MIN}
                NEW_EXPECTED=${PLATINUM_TB}
                NEW_PEAK=${PLATINUM_PEAK}
                ;;
              gold) 
                NEW_MIN=${GOLD_MIN}
                NEW_EXPECTED=${GOLD_TB}
                NEW_PEAK=${GOLD_PEAK}
                ;;
              silver) 
                NEW_MIN=${SILVER_MIN}
                NEW_EXPECTED=${SILVER_TB}
                NEW_PEAK=${SILVER_PEAK}
                ;;
              bronze) 
                NEW_MIN=${BRONZE_MIN}
                NEW_EXPECTED=${BRONZE_TB}
                NEW_PEAK=${BRONZE_PEAK}
                ;;
            esac

            echo "  SSHCMD ${CLUSTER} qos adaptive-policy-group create -vserver ${VSERVER} -policy-group ${NEW_QOSPOLGR} -absolute-min-iops ${NEW_MIN} -expected-iops ${NEW_EXPECTED} -peak-iops ${NEW_PEAK} -expected-iops-allocation allocated-space -peak-iops-allocation allocated-space"|tee -a ${LOG}
            SSHCMD "${CLUSTER} qos adaptive-policy-group create -vserver ${VSERVER} -policy-group ${NEW_QOSPOLGR} -absolute-min-iops ${NEW_MIN} -expected-iops ${NEW_EXPECTED} -peak-iops ${NEW_PEAK} -expected-iops-allocation allocated-space -peak-iops-allocation allocated-space"
            SSHCMD ${CLUSTER} "qos adaptive-policy-group show -vserver ${VSERVER}"|grep ${VSERVER}
          fi  # "${QOSPOLGR}" = "${NEW_QOSPOLGR}"

        fi  # "${AQOS_CLASS}" = ""
      fi  # SETAQOS


# Show (as last part) the policy
# And if MAIL, then do so & mail later
      if [ ${MAIL} ] || [ ${SHOWQOS} ] || [ ${SETAQOS} ]; then
        SSHCMD ${CLUSTER} "set -units GB -showseparator \";\"; volume show -vserver ${VSERVER} -volume ${VOL} -field vserver,volume,aggregate,total,type,qos-policy-group,qos-adaptive-policy-group,comment,create-time"|grep "${VSERVER}" >> ${TMP}
# Get info from cDOT 1st. Then get this in VAR back
      fi  # SHOWQOS

    done  # vol show
  done  # cat ${TMP}.2
done  # CLUSTER

if [ ${MAIL} ]; then
  echo "#" >> ${TMP}
  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" >> ${TMP}
  cp ${TMP} ${CSV}
  date | mailx -a ${CSV} -s "Set QoS policy [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
fi  # MAIL


rm ${TMP} ${TMP}.2 
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0