#!/bin/ksh
set +x
#--------------------------------------------------------------------------#
# Name: volgrow.sh                                                         #
# Description: Script to automate the resizing of volumes.                 #
# References:                                                              #
# Documentation:                                                           #
# Parameters: <none>                                                       #
# Usage: volgrow.sh                                                        #
# Global Description:                                                      #
# Author(s): AtoS                                                          #
# Change log:                                                              #
# aut   date            vers    comments                                   #
# RtB   04-05-2007      1.0     Initial version Ronald ten Broeke          #
# RL    06-11-2008      1.1     Update                                     #
# MdB	07-08-2009	1.2	NXP mod's and INODS added		   #
# MdB	27-08-2009	1.3	One LOGFILE = volgrow-shrink.log	   #
# MdB	29-09-2009	1.4	2>/dev/null added at "vol status" &	   #
#    				if [ `echo ${PCTGB} | grep '[0-9][0-9]'` ] #
#				&& [ ${PCTGB} -gt 89 ]; then		   #
#    				date +%Y%m%d corrected &		   #
#    				DEBUGLOG disabled		   	   #
# MdB	04-10-2009	1.5	DEBUGLOG enabled, because it looks 	   #
#				vol size + did not work accourding $LOG	   #
# MdB	09-05-2011	2.0	With USD messages about growed volumes	   #
# MdB	11-05-2011	2.1	Added AGGRMAX				   #
# MdB	12-05-2011	2.2	Changed LOGGER msg			   #
# MdB	13-05-2011	2.3	For 1 volume (due to an Incident)	   #
# MdB	13-05-2011	2.4	Vol's with fs_size_fixed=on can not grew   #
# RL	15-09-2011	2.5	Increase destination volume in SM first    #
# AH	19-11-2012	2.6	Bug fix for small volumes		   #
# MdB   08-03-2013      2.7     Add CI=                                    #
# RL	12-04-2013	2.8	Increase destination volume in SM first    #
# MdB	11-11-2014	3.0	Alert -> IMI. Added --imi, --nagios, LOGFILE=${PGM}.log
# MdB	14-11-2014	3.1	MSGSGR=$MSGSTR + ...
# MdB	12-01-2016	3.2	Added some LOG entries
# MdB	13-01-2016	3.3	Mod. some echo fs_size_fixed + SDMTYPE="Incident" SDMSEV="3"
#                                                                          #
#--------------------------------------------------------------------------#
# Initialize variabels                                                     #
#--------------------------------------------------------------------------#

PGM="`basename $0|cut -d\. -f1`"
TMP="/tmp/${PGM}.$$"
HOSTNAME="`hostname`"
VER="3.3"

TIME=`date +%Y%m%d%H%M`
LOGFILE="${HOME}/log/${PGM}.log"
##LOGFILE="${HOME}/log/volgrow-shrink.log"
EXCLFILE="${HOME}/etc/${PGM}.excl"
DEBUGLOG="${HOME}/log/${PGM}.debuglog"
FILTER="[?]*"
FILERS="${HOME}/etc/filers"
MSGSTR=""
SSH="/usr/bin/ssh -n"
RESIZE=""
VOLFILTER="vol"

NAGIOSSEV="WARNING"
NAGIOS=""
IMI=""
SDMMAIL="ESBIncident.it-solutions@atos.net"
#SDMMAIL="maarten.deboer@atos.net"
#SDMUSER="nl19471"
#SDMREPLYTO="maarten.deboer@atos.net"
SDMMSG="${TMP}.sdm"
SDMTYPE="Query"
SDMTEXT="${TMP}.sdmtext"
SDMSEV="4"
#SDMGROUP="NL.Storage.FSOD"
#SDMGROUP="PL.Storage.FSOD"
SDMGROUP="RO.Storage.FSOD"

SDMUSER="nldsm01"
SDMREPLYTO="fsod@atos.net"

#--------------------------------------------------------------------------#
# Functions								   #
#--------------------------------------------------------------------------#
CREATE_SDM_TICKET()
# CREATE_SDM_TICKET "${FILER}" "${SDMTYPE}" "${SDMTEXT}" "${SDMSEV}"
# $1=Filer(CI), $2=SDMTYPE, $3=SDM-text $4=SEV
{
  echo "@REQUESTTYPE: ${2}" > ${SDMMSG}
  echo "@REQUESTAREA: NL.Storage.StorageOnDemand" >> ${SDMMSG}
  echo "@SEVERITY: ${4}" >> ${SDMMSG}
  echo "@CI: ${1}" >> ${SDMMSG}
  echo "@ORGANISATION: AtosOrigin.Netherlands" >> ${SDMMSG}
  echo "@GROUP: ${SDMGROUP}" >> ${SDMMSG}
  echo "@REPLYTO: ${SDMREPLYTO}" >> ${SDMMSG}
  echo "@PROXYUSER: ${SDMUSER}"  >> ${SDMMSG}
  cat ${3} >> ${SDMMSG}
  echo "[@${HOSTNAME}:${PGM} v${VER}]" >> ${SDMMSG}
  echo "@ENDOFSDMMESSAGE" >> ${SDMMSG}

  cat ${SDMMSG} | mailx -s "${SDMUSER}: ${PGM} ${2} at ${1}." ${SDMMAIL}

  echo "`date` ${PGM}: Mailed to SDM-IMI (${SDMMAIL})"|tee -a ${LOGFILE}
}


SEND_TO_NAGIOS()
# SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
# $1=NAGIOSSEV $2=MSGSTR
{
# Nagios will be needed for every item (volume). Because limitation of 255 char's
# For more; IMI need to be used.
  if [ "${NAGIOS}" ]; then
  #----------------------------------------------------------------#
  # To Nagios (via logger)                                                         #
  # ------                                                         #
  # MINOR         Nagios-WARNING & SEV.3                           #
  # WARNING       Nagios-WARNING & SEV.3                           #
  # CRITICAL      Nagios-CRITICAL & SEV.2                          #
  # FATAL         Nagios-CRITICAL & SEV.2                          #
  # logger -t NOTIFY FSOD-SUPPORT <Nagios-SEV> "Text"              #
  #----------------------------------------------------------------#
  /usr/bin/logger -t NOTIFY FSOD-SUPPORT ${1} "${MSGSTR}"
# echo "logger: NOTIFY FSOD-SUPPORT ${1} ${2}"|tee -a ${LOGFILE}
  echo "logger: NOTIFY TEST-SUPPORT ${1} ${2}"|tee -a ${LOGFILE}
  fi  # [ "${NAGIOS}" ]; then
}

USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options       :"
  echo "    -f          : filter filername (${FILTER})"
  echo "    -h|--help   : this help"
  echo "    -i|--imi    : send alert (to SDM) via IMI(mail)"
  echo "    -n|--nagios : send (short)alert (to SDM) via logger+Nagios"
  echo "    -r          : do Resize"
  echo "    -v          : Volumefilter (${VOLFILER})"
  echo "    -V          : Version"
  echo "    -x          : set -x"
}

# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -f) FILTER=$2; shift ;;
    -v) VOLFILTER=$2; shift ;;
    -r) RESIZE=1;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    -i | --imi) IMI=1;;
    -n | --nagios) NAGIOS=1;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

#--------------------------------------------------------------------------#
# $EXCLFILE should exist.
#--------------------------------------------------------------------------#
if [ ! -f "${EXCLFILE}" ]
then
   echo "${TIME}: ${EXCLFILE} doens't exist, create empty one"
   >${EXCLFILE}
fi

#--------------------------------------------------------------------------#
echo "`date` ${PGM}: v${VER} @${HOSTNAME} Started with FILER=${FILTER} & VOLUME=${VOLFILTER}"| tee -a ${LOGFILE}
echo "FILTER=${FILTER}"
echo "IMI=${IMI}"
echo "NAGIOS=${NAGIOS}"
echo "TMP=${TMP}"
echo "PGM=${PGM}"
echo "RESIZE=${RESIZE}"
echo "SDMMAIL=${SDMMAIL}"
echo "SDMUSER=${SDMUSER}"
echo "SDMMSG=${SDMMSG}"
echo "SDMREPLYTO=${SDMREPLYTO}"
echo "SDMTYPE=${SDMTYPE}"
echo "SDMTEXT=${SDMTEXT}"
echo "SDMSEV=${SDMSEV}"
echo "SDMGROUP=${SDMGROUP}"
echo "VOLFILTER=${VOLFILTER}"
echo "VER=${VER}"

touch ${TMP} ${TMP}.sdm ${SDMTEXT}
for FILER in `cat ${FILERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${FILTER}"`
do
  # clean some file 1st
  cp /dev/null ${SDMTEXT}
  #------------------------------------------------------------------------#
  # Check filer for volumes that must be enlarged 			   #
  #------------------------------------------------------------------------#
  echo "Filer: ${FILER} ..."
  for VOLUME in `${SSH} ${FILER} vol status|grep ${VOLFILTER} 2>/dev/null | grep online | grep -v Volume | awk '{print $1}' | sort`
  do
    if [ -z `grep "${FILER}:${VOLUME}" ${EXCLFILE}` ]; then
      PCTGB=`${SSH} ${FILER} df -g ${VOLUME} | grep -v File | grep -v snap | awk '{print $5}' | cut -d% -f1`
#      PCTINODES=`${SSH} ${FILER} df -i ${VOLUME} | grep -v File | awk '{print $4}' | cut -d% -f1`
      #--------------------------------------------------------------------#
      # If the volume is more than 89% filled				   #
      # Test if $PCTGB is number.					   #
      # If the volume is less than 80% filled, 				   #
      # more than 100GB and not 0GB used. 				   #
      #--------------------------------------------------------------------#
      if [ `echo ${PCTGB} | grep '[0-9][0-9]'` ] && [ ${PCTGB} -gt 89 ]; then
        #------------------------------------------------------------------#
        # Set variables                                                    #
        #------------------------------------------------------------------#
        VOLOLD=`${SSH} ${FILER} df -g ${VOLUME} | grep -v File | grep -v snap | awk '{print $2}' | cut -dG -f1`
        VOLUSED=`${SSH} ${FILER} df -g ${VOLUME} | grep -v File | grep -v snap | awk '{print $3}' | cut -dG -f1`

        #------------------------------------------------------------------#
        # Lets make NEWVOL 85%						   #
        #------------------------------------------------------------------#
        let "VOLNEW= ${VOLUSED} * 100 / 85"
        if [ "${VOLNEW}" -le "${VOLOLD}" ]; then
        let "VOLNEW= ${VOLNEW} + 2"
        fi
        let "VOLDIFF= ${VOLNEW} - ${VOLOLD}"

        #------------------------------------------------------------------#
        # Check if enough space at aggregate				   #
        #------------------------------------------------------------------#
        AGGR=`${SSH} ${FILER} vol status ${VOLUME} | grep agg | cut -d\' -f2`
        AGGRS=`${SSH} ${FILER} df -Ag ${AGGR} | grep -v Aggr | grep -v snap | awk '{print $2}' | cut -dG -f1`
        AGGRU=`${SSH} ${FILER} df -Ag ${AGGR} | grep -v Aggr | grep -v snap | awk '{print $3}' | cut -dG -f1`
        AGGRPERC=`${SSH} ${FILER} df -Ag ${AGGR} | grep -v Aggr | grep -v snap | awk '{print $5}'`
        let "AGGRL= ${AGGRS} * 90 / 100"
        let "AGGRA= ${AGGRL} - ${AGGRU}"

        if [ "${VOLDIFF}" -lt "${AGGRA}" ]; then
          echo "`date` ${PGM}: Volume ${FILER}:${VOLUME} need to grow to ${VOLNEW}g (with ${VOLDIFF}) from ${VOLOLD}g. Used:${VOLUSED}g = ${PCTGB}% at Aggr=${AGGR} ${AGGRS}GB ${AGGRPERC}"|tee -a ${LOGFILE} ${DEBUGLOG}
          if [ "${RESIZE}" ]; then
            ${SSH} ${FILER} vol size ${VOLUME} >> ${DEBUGLOG}
            #--------------------------------------------------------------#
            # Test if fs_size_fixed=on (vol status)  			   #
            # If so, resize can not be done and can be a part of a SM      #
            #--------------------------------------------------------------#
            SF="`${SSH} ${FILER} vol options ${VOLUME}|grep fs_size_fixed|cut -d\= -f2|cut -d\, -f1`"
            echo "  ${FILER}:${VOLUME} SF=${SF}"|tee -a ${LOG}
            if [ "${SF}" = "on" ]; then  # fs_size_fixed=on
              echo "  ${FILER}:${VOLUME} fs_size_fixed=on (${SF}). So NO resize can be done."|tee -a ${LOG}
              SM=`${SSH} ${FILER} snapmirror status|grep ${VOLUME}|grep -v Source|grep -v "/vol/"`

              if [ "${SM}" != "" ]; then # Has a SnapMirror relation
                echo "  ${FILER}:${VOLUME} SM(${SM}) relations"|tee -a ${LOG}
                SM_SRC=`${SSH} ${FILER} snapmirror status | grep ${VOLUME} | grep -v Source | grep -v "/vol/" | awk '{print $1}'`
                SM_DST=`${SSH} ${FILER} snapmirror status | grep ${VOLUME} | grep -v Source | grep -v "/vol/" | awk '{print $2}'`
                SM_SRC_FILER=`echo ${SM_SRC} | awk '{print $1}' | awk -F":" '{print $1}' | awk -F"-" '{print $1}'`
                SM_SRC_VOLUME=`echo ${SM_SRC} | awk '{print $1}' | awk -F":" '{print $2}'`

	        if [ "${SM_DST}" = "${FILER}:${VOLUME}" ]; then
    	          #--------------------------------------------------------#
                  # Check if enough space at destination aggregate	   #
    	          #--------------------------------------------------------#
    	          AGGR_SRC=`${SSH} ${SM_SRC_FILER} vol status ${SM_SRC_VOLUME} | grep agg | cut -d\' -f2`
   	          AGGRS_SRC=`${SSH} ${SM_SRC_FILER} df -Ag ${AGGR_SRC} | grep -v Aggr | grep -v snap | awk '{print $2}' | cut -dG -f1`
   	          AGGRU_SRC=`${SSH} ${SM_SRC_FILER} df -Ag ${AGGR_SRC} | grep -v Aggr | grep -v snap | awk '{print $3}' | cut -dG -f1`
   	          AGGRPERC_SRC=`${SSH} ${SM_SRC_FILER} df -Ag ${AGGR_SRC} | grep -v Aggr | grep -v snap | awk '{print $5}'`
    	          let "AGGRL_SRC= ${AGGRS_SRC} * 90 / 100"
    	          let "AGGRA_SRC= ${AGGRL_SRC} - ${AGGRU_SRC}"

   	          if [ "${VOLDIFF}" -lt "${AGGRA_SRC}" ]; then
     	            echo "`date` ${PGM}: Volume ${SM_SRC_FILER}:${SM_SRC_VOLUME} need to grow to ${VOLNEW}g (with ${VOLDIFF}) from ${VOLOLD}g. Used:${VOLUSED}g = ${PCTGB}% at Aggr=${AGGR_SRC} ${AGGRS_SRC}GB ${AGGRPERC_SRC}"|tee -a ${LOGFILE} ${DEBUGLOG}
                    ${SSH} ${FILER} vol size ${VOLUME} ${VOLNEW}g >> ${DEBUGLOG}
                    echo "`date` resized ${FILER}:${VOLUME} to ${VOLNEW}g"|tee -a ${LOGFILE}
                    MSGSTR="CI=${FILER} INFORMATION: Volume ${FILER}:${VOLUME} grew to ${VOLNEW}g (with ${VOLDIFF}) from ${VOLOLD}g. Used:${VOLUSED}g = ${PCTGB}% at Aggr=${AGGR} ${AGGRS}GB ${AGGRPERC} [${PGM} v${VER}]"
                    echo ${MSGSTR} >> ${SDMTEXT}
                    SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
                    MSGSTR=""

	            ${SSH} ${SM_SRC_FILER} vol size ${SM_SRC_VOLUME} ${VOLNEW}g >> ${DEBUGLOG}
                    echo "`date` resized ${SM_SRC_FILER}:${SM_SRC_VOLUME} to ${VOLNEW}g"|tee -a ${LOGFILE}
                    MSGSTR="CI=${SM_SRC_FILER} INFORMATION: Volume ${SM_SRC_FILER}:${SM_SRC_VOLUME} grew to ${VOLNEW}g (with ${VOLDIFF}) from ${VOLOLD}g. Used:${VOLUSED}g = ${PCTGB}% at Aggr=${AGGR_SRC} ${AGGRS_SRC}GB ${AGGRPERC_SRC} [${PGM} v${VER}]"
                    echo ${MSGSTR} >> ${SDMTEXT}
                    SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
                    MSGSTR=""

                    SDMTYPE="Query"
                    SDMSEV="4"
	          fi  # [ "${VOLDIFF}" -lt "${AGGRA_SRC}" ]; then
                fi  # [ "${SM_DST}" = "${FILER}:${VOLUME}" ]; then
	      else # [ "${SM}" != "" ]; then # Has a SnapMirror relation
                MSGSTR="CI=${FILER} ATTENTION: Volume ${FILER}:${VOLUME} has fs_size_fixed=on. So can NOT be resized from ${VOLOLD}g to ${VOLNEW}g."
                echo "`date` ${PGM}: ${MSGSTR}"|tee -a ${LOGFILE}
                echo ${MSGSTR} >> ${SDMTEXT}
                # SDMTYPE="Query"
                SDMTYPE="Incident"
                SDMSEV="3"
                SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
                MSGSTR=""
	      fi  # [ "${SM}" != "" ]; then # Has a SnapMirror relation
            else  # [ "${SF}" = "on" ]; then  # fs_size_fixed=on
              # fs_size_fixed=off, so volume can be resized.
              ${SSH} ${FILER} vol size ${VOLUME} ${VOLNEW}g >> ${DEBUGLOG}
              echo "Resized ${FILER}:${VOLUME} to ${VOLNEW}g" |tee -a ${LOGFILE}
              MSGSTR="CI=${FILER} INFORMATION: Volume ${FILER}:${VOLUME} grew to ${VOLNEW}g (with ${VOLDIFF}) from ${VOLOLD}g. Used:${VOLUSED}g = ${PCTGB}% at Aggr=${AGGR} ${AGGRS}GB ${AGGRPERC} [${PGM} v${VER}]"
              echo ${MSGSTR} >> ${SDMTEXT}
              SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
              MSGSTR=""

              SDMTYPE="Query"
              SDMSEV="4"
            fi  # [ "${SF}" = "on" ]; then  # fs_size_fixed=o
          else  # [ "${RESIZE}" ]; then
            echo "RESIZE-cmd: ${FILER}:${VOLUME} to ${VOLNEW}g. NO resizing done. Option -r is needed for this."|tee -a ${SDMTEXT} ${LOGFILE}
            echo ${MSGSTR} >> ${SDMTEXT}
            SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
            MSGSTR=""

            SDMTYPE="Query"
            SDMSEV="3"
          fi  # [ "${RESIZE}" ]; then
        else
          echo "`date` ${PGM}: Aggregate ${FILER}:${AGGR} does not have enough space to extend ${VOLUME} with ${VOLDIFF} GB" | tee -a ${LOGFILE}
          MSGSTR="CI=${FILER} ATTENTION: Aggregate ${FILER}:${AGGR} does not have enough space to extend ${VOLUME} with ${VOLDIFF} GB [${PGM} v${VER}]"
          echo ${MSGSTR} >> ${SDMTEXT}
          SEND_TO_NAGIOS ${NAGIOSSEV} ${MSGSTR}
          MSGSTR=""

          SDMTYPE="Query"
          SDMSEV="3"
        fi  # [ "${VOLDIFF}" -lt "${AGGRA}" ]; then
      fi  # [ `echo ${PCTGB} | grep '[0-9][0-9]'` ] && [ ${PCTGB} -gt 89 ]; then
    fi  # [ -z `grep "${FILER}:${VOLUME}" ${EXCLFILE}` ]; then
  done  # for VOLUME in `${SSH} ${FILER} vol status|grep ${VOLFILTER} 
  if [ "${IMI}" ]; then
    if [ -s ${SDMTEXT} ]; then  # If size > 0
      CREATE_SDM_TICKET "${FILER}" "${SDMTYPE}" "${SDMTEXT}" "${SDMSEV}" 
      cp /dev/null ${SDMTEXT}
    fi  # [ -s ${SDMTEXT} ]; then  # If size > 0
  fi  # IMI
done  # for FILER in `cat ${FILERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${FILTER}"`

rm ${TMP} ${SDMMSG} ${SDMTEXT}
echo "`date` ${PGM}: v${VER} @${HOSTNAME} finished." | tee -a ${LOGFILE}
exit 0