#!/bin/ksh # File : cdot_set_qos_settings.sh # By : Maarten de Boer, 180116 # Subject : Script to set QoS settings # set -x # ToDos: # Cleaning up not used QoS-policies. But NOT the 0-INF ones # Check if right policies defined (_class_ ). # If not ERROR & mail #(0.4),180118 : Mod incident when no qos-policy-group at a volume #(0.5),180118 : Added --setsize) SETQOS=1 ;; #(0.6),180126 : Mod MAX_PT's #(0.7),180126 : Add --setpolicy for setting the INF policy to the volumes #(0.8),180126 : Added VFILTER #(0.9),180322 : Mod. No LS-volumes #(0.10),180522 : Changed (verbose) echo's # PGM="`basename $0|cut -d\. -f1`" VER="0.10" TMP="/tmp/${PGM}.$$" CLUSTERS="${HOME}/etc/clusters" MAILTO="maarten.deboer@atos.net" SSH="/usr/bin/ssh" HOSTNAME="`hostname | cut -d\. -f1`" LOG="${HOME}/log/${PGM}.log" FILTER="[?]*" MAIL="" SETQOS="" SETANYWAY="" SETPOLICY="" VERBOSE="" MONIDHDR="MAS.NL.1" SDMCLASS="ZZ-Event.Storage.Storage" ONTAPARCH="[cdot]" NAGIOS="" NAGIOSSEV="WARNING" NAGIOSSEVNR="1" NAGIOSPASSIVEDIR="/appl/dfm/nagios/PassiveCheck/DVL" #NAGIOSPASSIVEDIR="/appl/dfm/nagios/PassiveCheck/PRD" # VPREFIX needed at 'vserver show' VPREFIX="nl" # QoS -max-throughput MAXQTP="INF" # Storage classes # Diamond | : 5000 IOPS /TB (1-2ms) # Platinum | First (FC) : 1500 IOPS /TB (3-5 ms) # Gold | Business (BC) : 500 IOPS /TB (4-8 ms) # Silver | Economy (EC) : 250 IOPS /TB (6-12 ms) # Bronze | Archive (AC) : 25 IOPS /TB (8-28 ms) # MAX_PT = Max percentage as MAX QoS STORAGECLASSES="diamond platinum gold silver bronze" DIAMONT_TB=5000 DIAMONT_MAX_PT=200 PLATINUM_TB=1500 PLATINUM_MAX_PT=200 GOLD_TB=500 GOLD_MAX_PT=150 SILVER_TB=250 SILVER_MAX_PT=150 BRONZE_TB=25 BRONZE_MAX_PT=100 # AGGRlist /CLASS AGGRGOLD="n[0-9][0-9]a[01][62][0-9][0-9]" AGGRSILVER="n[0-9][0-9]a20[0-9][0-9]" AGGRBRONZE="n[0-9][0-9]a[34]0[0-9][0-9]" SSHCMD() # 1: Filername 2:Command-string # When issue with connection to cluster, try the nodes (-01 & -02) # "There are no entries matching your query." => EC=255 # "no connection" is also EC=255 { TMPERR="/tmp/${PGM}.$$.err" touch ${TMPERR} /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR} EC=${?} # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 1 /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 1 /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 1 /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 1 /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 1 /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 1 /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then echo "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} fi # EC=0 & EC2=0 fi # -06 fi # -05 fi # -04 fi # -03 fi # -02 fi # -01 rm ${TMPERR} } USAGE() { echo "Usage: ${PGM} " echo " Version: ${VER}" echo " options :" echo " -e|--etc : Etc/clusters-file (${CLUSTERS})" echo " -f : Filter clustername (${FILTER})" echo " -h|--help : this Help" echo " -m|--mail : do send Mail" echo " -p : set vPrefix (for Vserver / SVM) (${VPREFIX})" echo " -v|--verbose : set Verbose" echo " -V : show Version" echo " -x : set -x" echo " --mailto : change MAILTO address & do send mail (${MAILTO})" echo " --setqos : do SETQOS" echo " --setanyway : do set QOS anyway (needed when definitions are changed)" echo " --setpolicy : Set the INF (max) policy to the volumes" } # Check options if [ $# -eq 0 ]; then echo " No options give. Do not know what to do. Exiting ..." USAGE exit 3 fi while [ $# -gt 0 ] do case $1 in -e | --etc) CLUSTERS=$2; shift ;; -f) FILTER=$2; shift ;; -p) VPREFIX=$2; shift ;; -v | --verbose) VERBOSE=1 ;; -m | --mail) MAIL=1 ;; -n | --nagios) NAGIOS=1 ;; --mailto) MAILTO=$2; MAIL=1; shift ;; -h | --help) USAGE; exit 1 ;; -V) echo "${PGM}: v${VER}"; exit 3 ;; -x) set -x ;; --setqos) SETQOS=1 ;; # Added setanyway. Vol size change is not taken in account. # Needed when definitions are changed. Then qos need to be set. --setanyway) SETANYWAY=1 ;; --setpolicy) SETPOLICY=1 ;; *) echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;; esac shift done # case # MAIN echo "`date` ${PGM} v${VER} started (CLUSTERS=${CLUSTERS},FILTER=${FILTER},MAIL=${MAIL},NAGIOS=${NAGIOS},SETQOS=${SETQOS},SETANYWAY=${SETANYWAY},SETPOLICY=${SETPOLICY},VPREFIX=${VPREFIX})"|tee -a ${LOG} touch ${TMP} echo "#; " > ${TMP} cat "${CLUSTERS}"|grep -v \^#|awk -F\; '{print $1}'|sort|grep "${FILTER}"|while read CLUSTER do # echo " ${CLUSTER} ..." # Getting the nodes 1st SSHCMD ${CLUSTER} "vserver show -operational-state running"|grep ${VPREFIX}|awk '{print $1}' > ${TMP}.2 cat ${TMP}.2|while read VSERVER do # The 1st line (without LF) can be : # ssh: connect to host nlnaf100-01 port 22: Connection refused # The 2nd line can be: # nlnaf100-04 205 days 02:54 # So need the check this out #echo "1st SSHCMD:" echo " ${CLUSTER}/${VSERVER}..." # Check if policy-groups /SVM are created by --setpolicy if [ ${SETPOLICY} ]; then for CLASS in ${STORAGECLASSES} do MAXTHP=`SSHCMD ${CLUSTER} "qos policy-group show -vserver ${VSERVER} -policy-group ${VSERVER}_${CLASS} -field max-throughput"|grep ${VSERVER}|awk '{print $2}'` if [ ${VERBOSE} ]; then echo " policy-group:${VSERVER}_${CLASS}=${MAXTHP}" fi if [ "${MAXTHP}" != "INF" ]; then echo " NO policy-group set for ${VSERVER} ${CLASS}"|tee -a ${LOG} if [ ${VERBOSE} ]; then echo " SSHCMD ${CLUSTER} qos policy-group create -vserver ${VSERVER} -policy-group ${VSERVER}_${CLASS} -max-throughput INF"|tee -a ${LOG} fi # VERBOSE SSHCMD ${CLUSTER} "qos policy-group create -vserver ${VSERVER} -policy-group ${VSERVER}_${CLASS} -max-throughput INF" fi # MAXTHP done # CLASS fi # SETPOLICY # Leave out SVM-root-vols (with "root" in the volume name) & LS-volume (with "vol0ls" in the volume name) SSHCMD ${CLUSTER} "set -units GB -showseparator \";\"; vol show -vserver ${VSERVER} -field aggregate,qos-policy-group,total"|grep ${VPREFIX}|egrep -v 'root|vol0ls'|while read LINE do # echo ${LINE} VOL=`echo ${LINE}|awk -F\; '{print $2}'` AGGR=`echo ${LINE}|awk -F\; '{print $3}'` TOTAL=`echo ${LINE}|awk -F\; '{print $4}'` QOSPOLGR=`echo ${LINE}|awk -F\; '{print $5}'` QOS_CLASS=`echo ${LINE}|awk -F\; '{print $5}'|cut -d\_ -f2` QOS_SIZE=`echo ${LINE}|awk -F\; '{print $5}'|cut -d\_ -f3` if [ ${VERBOSE} ]; then echo " ${VSERVER}:${AGGR}/${VOL}=${TOTAL} | ${QOSPOLGR} (${QOS_CLASS}_${QOS_SIZE})" fi if [ "${QOSPOLGR}" = "-" ]; then # If no qos set, then find out class 1st SCLASS="" VOLISBRONZE=`echo ${AGGR}|grep ${AGGRBRONZE}` VOLISSILVER=`echo ${AGGR}|grep ${AGGRSILVER}` VOLISGOLD=`echo ${AGGR}|grep ${AGGRGOLD}` if [ "${VOLISBRONZE}" != "" ]; then SCLASS="bronze" fi if [ "${VOLISSILVER}" != "" ]; then SCLASS="silver" fi if [ "${VOLISGOLD}" != "" ]; then SCLASS="gold" fi # If --setpolicy, then set a INF policy onto a volume according the aggr. if [ ${SETPOLICY} ]; then if [ ${VERBOSE} ]; then echo " SSHCMD ${CLUSTER} vol modify -vserver ${VSERVER} -volume ${VOL} -qos-policy-group ${VSERVER}_${SCLASS}"|tee -a ${LOG} fi # VERBOSE SSHCMD ${CLUSTER} "vol modify -vserver ${VSERVER} -volume ${VOL} -qos-policy-group ${VSERVER}_${SCLASS}" fi # SETPOLICY # NO QoS-policy set. Generate an incident. WARNINGSTR="No qos-policy-group set at volume ${CLUSTER}/${VSERVER}:${VOL}" echo " ${WARNINGSTR}"|tee -a ${LOG} if [ ${MAIL} ]; then echo "${WARNINGSTR}"|mailx -s "No QoS-policy set [${PGM} v${VER}]" ${MAILTO} echo " ${WARNINGSTR}. Mailed to ${MAILTO}"|tee -a ${LOG} fi # MAIL if [ ${NAGIOS} ]; then CI="${CLUSTER}" # Check for cDOT-node. If not, then -PLF ANSW="`echo ${CI}|grep 'nlnaf1[0-9][0-9]-[0-9][0-9]'`" if [ "${ANSW}" != "" ]; then MONID="${MONIDHDR}.${CI}-PLF" fi DATI="`date +%Y-%m-%d-%H-%M-%S`" NAGIOSFILE="${NAGIOSPASSIVEDIR}/${DATI}" MSGTEXTSTR="CI=${CI} ${ONTAPARCH} ${WARNINGSTR} (${DATI}@${HOSTNAME}).|MONID=${MONID};CLASS=${SDMCLASS};" echo "${NAGIOSSEVNR}|${MSGTEXTSTR} (${DATI}).|MONID=${MONID};CLASS=${SDMCLASS};"|tee -a ${NAGIOSFILE} ${LOG} echo " ${WARNINGSTR}. Send to NAGIOS (${NAGIOSFILE})"|tee -a ${LOG} fi # NAGIOS fi # ${QOSPOLGR}" = "-" if [ ${SETANYWAY} ] || ( [ "${QOSPOLGR}" != "-" ] && ( [ "${QOS_SIZE}" = "" ] || [ "${TOTAL}" != "${QOS_SIZE}" ] ) ); then # If a QoS-pol (not "-") and empty QoS-size / QoS <> vol-total-size # Define a new name (QOS_NEWGROUP) QOS_NEWGROUP="${VSERVER}_${QOS_CLASS}_${TOTAL}" if [ ${VERBOSE} ]; then echo " --setanyway=(${SETANYWAY}) OR size NOT set OR Vol-size NOT same as QoS-size. Should be set to ${QOS_NEWGROUP}" fi MAXQTP="INF" # Calculate NEWGROUP # let PERC="(${TTLCNT}-${WARNCNT})*100/${TTLCNT}" # If MAXQTP > TB-value then set TB-value TSIZE=`echo ${TOTAL}|sed 's/GB//g'` case ${QOS_CLASS} in diamond) let MAXQTP="( ${TSIZE} * ${DIAMOND_TB} * (${DIAMOND_MAX_PT}/100) ) / 1024 " if [ ${MAXQTP} -lt ${DIAMOND_TB} ]; then MAXQTP=${DIAMOND_TB} fi ;; platinum) let MAXQTP="( ${TSIZE} * ${PLATINUM_TB} * (${PLATINUM_MAX_PT}/100) ) / 1024 " if [ ${MAXQTP} -lt ${PLATINUM_TB} ]; then MAXQTP=${PLATINUM_TB} fi ;; gold) let MAXQTP="( ${TSIZE} * ${GOLD_TB} * (${GOLD_MAX_PT}/100) ) / 1024 " if [ ${MAXQTP} -lt ${GOLD_TB} ]; then MAXQTP=${GOLD_TB} fi ;; silver) let MAXQTP="( ${TSIZE} * ${SILVER_TB} * (${SILVER_MAX_PT}/100) ) / 1024 " if [ ${MAXQTP} -lt ${SILVER_TB} ]; then MAXQTP=${SILVER_TB} fi ;; bronze) let MAXQTP="( ${TSIZE} * ${BRONZE_TB} * (${BRONZE_MAX_PT}/100) ) / 1024 " if [ ${MAXQTP} -lt ${BRONZE_TB} ]; then MAXQTP=${BRONZE_TB} fi ;; *) MAXQTP="INF" ;; esac if [ ${VERBOSE} ]; then echo " New QoS: ${VSERVER}:${VOL} QoS:${QOS_NEWGROUP} = ${MAXQTP} (IOPS)"| tee -a ${LOG} fi # Check 1st if (new)policy-group exists CHK_NEWGROUP=`SSHCMD ${CLUSTER} "qos policy-group show -vserver ${VSERVER} -policy-group ${QOS_NEWGROUP} -field policy-group"|grep ${QOS_NEWGROUP}` if [ "${CHK_NEWGROUP}" = "" ]; then # NO NEW group, so create if [ ${SETQOS} ] || [ ${SETANYWAY} ]; then if [ ${VERBOSE} ]; then echo " SSHCMD ${CLUSTER} qos policy-group create -vserver ${VSERVER} -policy-group ${QOS_NEWGROUP} -max-throughput ${MAXQTP} "|tee -a ${LOG} fi # VERBOSE SSHCMD ${CLUSTER} "qos policy-group create -vserver ${VSERVER} -policy-group ${QOS_NEWGROUP} -max-throughput ${MAXQTP}" fi else if [ ${SETQOS} ] || [ ${SETANYWAY} ]; then if [ ${VERBOSE} ]; then echo " SSHCMD ${CLUSTER} qos policy-group modify -policy-group ${QOS_NEWGROUP} -max-throughput ${MAXQTP}"|tee -a ${LOG} fi SSHCMD ${CLUSTER} "qos policy-group modify -policy-group ${QOS_NEWGROUP} -max-throughput ${MAXQTP}" fi fi # CHK_NEWGROUP # Set NEW QoS-policy onto volume if [ ${SETQOS} ] || [ ${SETANYWAY} ]; then if [ ${VERBOSE} ]; then echo " SSHCMD ${CLUSTER} volume modify -vserver ${VSERVER} -volume ${VOL} -qos-policy-group ${QOS_NEWGROUP}"|tee -a ${LOG} fi SSHCMD ${CLUSTER} "volume modify -vserver ${VSERVER} -volume ${VOL} -qos-policy-group ${QOS_NEWGROUP}" fi else # Nothing to be done echo " ." fi # [ "${QOS_SIZE}" = "" ] || [ "${TOTAL}" != "${QOS_SIZE}" ] done # vol show done # cat ${TMP}.2 done # CLUSTER rm ${TMP} ${TMP}.2 echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG} exit 0