#!/bin/bash # File : cdot_set_autosizing.sh # By : Maarten.deBoer@atos.net, 141222 # Subject : Script to set autosize parameters # About autosizing #(0.2),160713 : Added SSHCMD, set autosizing #(0.3),160713 : Set min-autosize according to SUm of the quota TOTALLIMIT #(0.4),160830 : Add usage & noQuota, use curr-size (in case of SV) #(0.5),161212 : Added an extra SET shrink-threshold-percent = 1 #(0.6),170208 : Added -f filter #(0.7),170608 : Added volume -type RW #(0.8),170614 : Added VOLFILTER #(0.9),170615 : Added DP volumes. Is no Quota to be gathered. But need to be set as well #(0.10),170921 : Mod. sizes #(0.11),170921 : Mod. show & then set. No set of DP vol-type (can not be set). Added SUMQUOTAPERC #(0.12),170921 : Added CONF-file. Less echo's in TMP-file #set -x PGM="`basename $0|cut -d\. -f1`" VER="0.12" TMP="/tmp/${PGM}.$$" ASC="/tmp/${PGM}.asc" LOG="${HOME}/log/${PGM}.log" CONF="${HOME}/etc/${PGM}.conf" SSH="/usr/bin/ssh -n" HOSTNAME="`hostname|cut -d\. -f1`" CLUSTERS="${HOME}/etc/clusters" PREFIX="nlnaf" CUSTCODE="" SETSIZE="" MAIL="" #MAILTO="maarten.deboer@atos.net" MAILTO="fsod@atos.net" FILTER="[?]*" VOLFILTER="" # Sum of the Quota perc (in 100%) for -min-autosize SUMQUOTAPERC=100 # Set autosize according discussion between Harold Kuijpers, Geralts Somsen & Maarten de Boer at 2016-06-07. # For SAP filers # Class Volume Size Max.size Min.size Grow Shrink Increment size (9.1 not any more) # Small 0B <= volume size < 250GB 1000GB 90% Sum.Quota 85% 75% 1GB # Medium 250GB <= volume size < 750GB 3000GB Idem 90% 80% 5GB # Large =>750GB +5TB Idem 93% 85% 10GB # Archiving (DP) # Archiving 102400GB 5GB 93% 80% 5GB SSHCMD() # 1: Filername 2:Command-string # When issue with connection to cluster, try the nodes (-01 & -02) # "There are no entries matching your query." => EC=255 # "no connection" is also EC=255 { TMPERR="/tmp/${PGM}.$$.err" touch ${TMPERR} /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR} EC=${?} # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 2 /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 2 /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then sleep 2 /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then echo " EC=${EC} sleep 2 ..." sleep 2 /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then echo " EC=${EC} sleep 2 ..." sleep 2 /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then echo " EC=${EC} sleep 2 ..." sleep 2 /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR} EC=${?} grep 'Connection refused' ${TMPERR} EC2=${?} if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then echo "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} fi # EC=0 & EC2=0 fi # -06 fi # -05 fi # -04 fi # -03 fi # -02 fi # -01 rm ${TMPERR} } USAGE() { echo "Usage: ${PGM} [options]" echo " Version: ${VER}" echo " options :" echo " -c : Customer code (3 lower-case letters)" echo " -e : Etc/clusters file (${CLUSTERS})" echo " -f : Filter (cluster) (${FILTER})" echo " -m | --mail : Mail the logging" echo " -h | --help : this help" echo " -v : Volfilter (${VOLFILTER})" echo " -V : Version" echo " -x : set -x" echo " --mailto : set new MAILTO (${MAILTO})" echo " --setsize : do SETSIZE" } # Check options while [ ${#} -gt 0 ] do case ${1} in -c) CUSTCODE=${2}; shift ;; -e) CLUSTERS=${2}; shift ;; -f) FILTER=${2}; shift ;; -v) VOLFILTER=${2}; shift ;; -m | --mail) MAIL=1 ;; -h | --help) USAGE; exit 1 ;; -V) echo "${PGM}: v${VER}"; exit 3 ;; -x) set -x ;; --mailto) MAIL=1; MAILTO=${2}; shift ;; --setsize) SETSIZE=1 ;; *) echo "Option ${1} not known."; USAGE; exit 1 ;; esac shift done if [ -z "${CUSTCODE}" ] then echo "`date` ${PGM}: ERROR; No (part of) customer code given. So not to know what to do. Exiting..."|tee -a ${LOG} ${ERR} ; echo; USAGE; exit 1 fi CUSTOMER=`echo "${CUSTCODE}"|tr [:lower:] [:upper:]` # MAIN touch ${TMP} echo "`date` ${PGM} v${VER} started."|tee -a ${LOG} ${TMP} # VOLFILTER need to have something. So if empty, then will be CUSTCODE if [ "${VOLFILTER}" = "" ]; then VOLFILTER=${CUSTCODE} fi if [ -f ${CONF} ]; then echo " CONF (${CONF}) found. Reading it ..."|tee -a ${LOG} . ${CONF} fi echo "CLUSTERS=${CLUSTERS}" echo "CUSTCODE=${CUSTCODE}" echo "FILTER=${FILTER}" echo "MAIL=${MAIL}" echo "MAILTO=${MAILTO}" echo "SETSIZE=${SETSIZE}" echo "SUMQUOTAPERC=${SUMQUOTAPERC}" echo "VOLFILTER=${VOLFILTER}" sleep 1 # NEWMINAS = min-autosize # NEWAGTP = -grow-threshold-percent # NEWASTP = -shrink-threshold-percent ${NEWASTP} # NEWIS = -increment-size # VOLMINAS = -minimum-size # VOLMAXAS = -maximum-size for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep ${FILTER}` do # echo "${CLUSTER} ..." # option -field selects the items to be shown. NOT the order. That is FIXED SSHCMD ${CLUSTER} 'set -showseparator " " ; vserver show -type data -fields vserver'|grep -v server|grep ${CUSTCODE}|while read VSERVER REST do echo "${CLUSTER}/${VSERVER}:" SSHCMD ${CLUSTER} "volume show -vserver ${VSERVER} -volume *${VOLFILTER}* -state online -fields volume"|egrep -v 'volume|vol0root|vol0ls'|grep ${PREFIX}|while read VSERVER VOLUME REST do VOLUSED="`SSHCMD ${CLUSTER} \"set -unit GB ; volume show -vserver ${VSERVER} -volume ${VOLUME} -fields used \" |grep ${VOLUME}|awk '{print $3}'|sed 's/GB//g'`" VOLSIZE="`SSHCMD ${CLUSTER} \"set -unit GB ; volume show -vserver ${VSERVER} -volume ${VOLUME} -fields size \" |grep ${VOLUME}|awk '{print $3}'|sed 's/GB//g'`" # Want to have the min-autosize used size for determing the growsizes # PREFIX is needed for getting only the amount of GB's and no other information VOLMINAS="`SSHCMD ${CLUSTER} \"set -unit GB ; volume show -vserver ${VSERVER} -volume ${VOLUME} -fields min-autosize \" |grep ${VOLUME}|awk '{print $3}'|sed 's/GB//g'`" VOLMAXAS="`SSHCMD ${CLUSTER} \"set -unit GB ; volume show -vserver ${VSERVER} -volume ${VOLUME} -fields max-autosize \" |grep ${VOLUME}|awk '{print $3}'|sed 's/GB//g'`" VOLTYPE="`SSHCMD ${CLUSTER} \"volume show -vserver ${VSERVER} -volume ${VOLUME} -fields type \" |grep ${VOLUME}|awk '{print $3}'`" echo "= ${CLUSTER}:${VSERVER}:${VOLUME}: size=${VOLSIZE} used=${VOLUSED} min-autosize=${VOLMINAS} max-autosize=${VOLMAXAS} type=${VOLTYPE}"|tee -a ${TMP} # # Get Qtree's & quotas & set min-autosize TOTALLIMIT=0 NEWMINAS="${VOLMINAS}" # When VOLTYPE=RW, then get sum of Quota if [ "${VOLTYPE}" = "RW" ]; then # Get Sum of Quota (only at RW type SSHCMD ${CLUSTER} "qtree show -vserver ${VSERVER} -volume ${VOLUME} -fields qtree"|grep ${PREFIX}|awk '{print $3}'|while read QTREE REST do if [ "${QTREE}" != "\"\"" ]; then DISKUSED="`SSHCMD ${CLUSTER} "set -units GB ; quota report -vserver ${VSERVER} -volume ${VOLUME} -tree ${QTREE} -field vserver,volume,tree,disk-used"|grep ${PREFIX}|awk '{print $5}'|sed 's/GB//g'`" if [ "${DISKUSED}" = "" ]; then DISKUSED=0 fi DISKLIMIT="`SSHCMD ${CLUSTER} "set -units GB ; quota report -vserver ${VSERVER} -volume ${VOLUME} -tree ${QTREE} -field vserver,volume,tree,disk-limit"|grep ${PREFIX}|awk '{print $5}'|sed 's/GB//g'`" if [ "${DISKLIMIT}" = "" ]; then DISKLIMIT=0 fi if [ ${DISKLIMIT} -gt 0 ]; then TOTALLIMIT=`expr ${TOTALLIMIT} + ${DISKLIMIT}` fi echo " ${VOLUME}/${QTREE}; disk-used=${DISKUSED}GB; disk-limit=${DISKLIMIT}GB; TOTALLIMIT=${TOTALLIMIT}GB" ##|tee -a ${TMP} fi # ${QTREE}" ! = "" # To get the value out site the subshell-loop echo ${TOTALLIMIT} > ${TMP}.TOTALLIMIT done # Qtree TOTALLIMIT="`cat ${TMP}.TOTALLIMIT`" if [ ${TOTALLIMIT} -gt 0 ]; then # if TOTALLIMIT > 0, then NEWMINAS is TOTALLIMIT = Sum of the Quotas # NEW: SUMQUOTAPERC of the Sum of the quota NEWMINAS=`expr ${TOTALLIMIT} \* ${SUMQUOTAPERC} / 100` fi fi # WR echo " ${CLUSTER}:${VSERVER}:${VOLUME}: VOLMINAS=${VOLMINAS}| TOTALLIMIT=${TOTALLIMIT}| "|tee -a ${TMP} # If TOTALLIMIT=0 (no quotas set), then set NEWMINAS to curr VOLUSED if [ ${TOTALLIMIT} -eq 0 ]; then NEWMINAS="${VOLUSED}" echo " ${CLUSTER}:${VSERVER}:${VOLUME}: TOTALLIMIT(SUM quota) was 0 => NEWMINAS=${NEWMINAS}GB"|tee -a ${TMP} fi # If VolType=DP, then NEWMINAS = 5GB # if [ "${VOLTYPE}" = "DP" ]; then # NEWMINAS="5" # echo " ${CLUSTER}:${VSERVER}:${VOLUME}: VOLTYPE=DP (SnapMirror/Vault) => NEWMINAS=${NEWMINAS}GB"|tee -a ${TMP} # fi # If NEWMINAS = 0 then set to used size if [ ${NEWMINAS} -eq 0 ]; then NEWMINAS="${VOLUSED}" echo " ${CLUSTER}:${VSERVER}:${VOLUME}: minimum-size was 0 => NEWMINAS=${NEWMINAS}GB"|tee -a ${TMP} fi # ${TOTALLIMIT} -lt 0 # Defaults NEWASMO="grow_shrink" NEWAGTP="93" NEWASTP="75" NEWIS="1" NEWMAXAS="${VOLMAXAS}" if [ ${VOLMINAS} -lt 250 ]; then NEWAGTP="85" NEWASTP="75" NEWIS="1" if [ ${VOLUSED} -lt 1000 ]; then NEWMAXAS="1000" fi fi # < 250 GB if [ ${VOLMINAS} -gt 250 ] && [ ${VOLMINAS} -lt 750 ] ; then NEWAGTP="90" NEWASTP="80" NEWIS="5" if [ ${VOLUSED} -lt 3000 ]; then NEWMAXAS="3000" fi fi # 250GB > size > 750 GB if [ ${VOLMINAS} -gt 750 ]; then NEWAGTP="93" NEWASTP="85" NEWIS="10" if [ ${VOLUSED} -lt 5000 ]; then NEWMAXAS="5000" else # *1,2 ## NEWMAXAS="`expr ${VOLSIZE} \* 12 / 10`" NEWMAXAS=${VOLMAXAS} fi fi # > 750 GB # Check some figures if [ ${NEWMINAS} -eq 0 ]; then NEWMINAS=1 fi # If NEWMINAS > NEWMAXAS, then NEWMAXAS = NEWMINAS+20% if [ ${NEWMINAS} -gt ${NEWMAXAS} ]; then NEWMAXAS="`expr ${NEWMINAS} \* 12 / 10`" fi # If NEWMAXAS < SIZE, then NEWMAXAS = SIZE+20% if [ ${NEWMAXAS} -lt ${VOLSIZE} ]; then NEWMAXAS="`expr ${VOLSIZE} \* 12 / 10`" fi # NEW: When Archive (=DP) then NEWMAXAS=102400GB # if [ "${VOLTYPE}" = "DP" ]; then # NEWMAXAS="102400" # fi # DP NEWIS="${NEWIS}GB" NEWMINAS="${NEWMINAS}GB" NEWMAXAS="${NEWMAXAS}GB" # Set when VOLTYPE=RW | DP (others not) # DP can NOT be set if [ "${VOLTYPE}" = "RW" ]; then # vserver volume max-autosize min-autosize autosize-grow-threshold-percent autosize-shrink-threshold-percent autosize-mode CURAUTOSIZE=`SSHCMD ${CLUSTER} "set -unit GB -showseparator \";\" ; volume show -vserver ${VSERVER} -volume ${VOLUME} -field autosize-grow-threshold-percent, autosize-shrink-threshold-percent, autosize-mode, min-autosize, max-autosize"| grep ${VSERVER}` CURMAXAS=`echo ${CURAUTOSIZE}|awk -F\; '{print $3}'` CURMINAS=`echo ${CURAUTOSIZE}|awk -F\; '{print $4}'` CURAGTP=`echo ${CURAUTOSIZE}|awk -F\; '{print $5}'|sed 's/%//g'` CURASTP=`echo ${CURAUTOSIZE}|awk -F\; '{print $6}'|sed 's/%//g'` CURASMO=`echo ${CURAUTOSIZE}|awk -F\; '{print $7}'` echo " Cur. CURMAXAS=${CURMAXAS} CURMINAS=${CURMINAS} CURAGTP=${CURAGTP} CURASTP=${CURASTP} CURASMO=${CURASMO} " |tee -a ${LOG} ${TMP} echo " New. NEWMAXAS=${NEWMAXAS} NEWMINAS=${NEWMINAS} NEWAGTP=${NEWAGTP} NEWASTP=${NEWASTP} NEWASMO=${NEWASMO} " |tee -a ${LOG} ${TMP} if [ "${CURMAXAS}" != "${NEWMAXAS}" ] || [ "${CURMINAS}" != "${NEWMINAS}" ] || [ "${CURAGTP}" != "${NEWAGTP}" ] || [ "${CURASTP}" != "${NEWASTP}" ] || [ "${CURASMO}" != "${NEWASMO}" ]; then echo " Not Equal. Setting ..."|tee -a ${LOG} ${TMP} sleep 1 if [ ${SETSIZE} ]; then echo "SET ${CLUSTER}:${VSERVER}:${VOLUME}: -grow-threshold-percent ${NEWAGTP} -shrink-threshold-percent ${NEWASTP} -increment-size ${NEWIS} -minimum-size ${NEWMINAS} -maximum-size ${NEWMAXAS} -mode ${NEWASMO}"|tee -a ${LOG} ${TMP} # Add an extra "volume autosize -shrink-threshold-percent 1" if "-shrink-threshold-percent" is higher then "-grow-threshold-percent" SSHCMD ${CLUSTER} "volume autosize -vserver ${VSERVER} -volume ${VOLUME} -shrink-threshold-percent 1" SSHCMD ${CLUSTER} "volume autosize -vserver ${VSERVER} -volume ${VOLUME} -grow-threshold-percent ${NEWAGTP} -shrink-threshold-percent ${NEWASTP} -increment-size ${NEWIS} -minimum-size ${NEWMINAS} -maximum-size ${NEWMAXAS} -mode ${NEWASMO} " else echo "NOT ${CLUSTER}:${VSERVER}:${VOLUME}: -grow-threshold-percent ${NEWAGTP} -shrink-threshold-percent ${NEWASTP} -increment-size ${NEWIS} -minimum-size ${NEWMINAS} -maximum-size ${NEWMAXAS} -mode ${NEWASMO}"|tee -a ${LOG} ${TMP} fi # ${SETSIZE} fi # Not Equal fi # WR done # volume done # vserver done # for CLUSTER if [ ${MAIL} ]; then echo "`date` Mailing to ${MAILTO}"|tee -a ${LOG} ${TMP} cp ${TMP} ${ASC} date | mailx -a ${ASC} -s ":${HOSTNAME}: Logging of cDOT set autosizing [${PGM} v${VER}]" ${MAILTO} rm ${ASC} fi # MAIL # Finish rm ${TMP} ${TMP}.TOTALLIMIT echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG} exit 0