#!/bin/bash
# File	: proc_fsod-change.sh
# By	: Maarten.deBoer@Atos.net, 180207
# Subject	: Script to process FSOD-changes
#(0.2),180208	: Add's Qtree delete
#(0.3),180208	: Add Quota create
#(0.4),180214	: Add qtree-show
#set -x
PGM=`basename $0|cut -d\. -f1`
VER="0.4"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
ASC="/tmp/${PGM}.asc"
ERR="/tmp/${PGM}.err"

RCFILE="/tmp/fsod-change.rc"
CLUSTER_VSERVER_FILE="/home/aodfm01a/export/fsod-clusters-vservers.csv"

FSOD_MGTSRV="aoxnlfsd01"
CLUSTER="nlnaf100"


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG}
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

# MAIN
echo "`date` ${PGM} v${VER} started." >> ${LOG}
touch ${TMP} ${ERR}

if [ -f ${RCFILE} ]; then
  . ${RCFILE}
else
  echo "  NO RCfile (${RCFILE}). So exiting ..."|tee -a ${LOG}
  exit 4
fi

echo "  With DATETIMESTAMP=${DATETIMESTAMP}, CHANGETYPE=${CHANGETYPE},CHANGENUMBER=${CHANGENUMBER},CLUSTERNAME=${CLUSTERNAME},SVMNAME=${SVMNAME},MAILADDRESS=${MAILADDRESS}"|tee -a ${LOG}

if [ "${SVMNAME}" != "" ] && [ "${CLUSTERNAME}" = "" ]; then
  CLUSTERNAME=`grep ${SVMNAME} ${CLUSTER_VSERVER_FILE}|cut -d\; -f1`
fi  # CLUSTERNAME

echo "CLUSTERNAME=${CLUSTERNAME}"
echo "SVMNAME=${SVMNAME}"
echo "CHANGENUMBER=${CHANGENUMBER}"
echo "CHANGETYPE=${CHANGETYPE}"
echo "VOLNAME=${VOLNAME}"
echo "QTREENAME=${QTREENAME}"
echo "QTREESTYLE=${QTREESTYLE}"
echo "QUOTASIZECHANGE=${QUOTASIZECHANGE}"
echo "QUOTADELTASIZEGB=${QUOTADELTASIZEGB}"
echo "MAILADDRESS=${MAILADDRESS}"
echo "DATETIMESTAMP=${DATETIMESTAMP}"
echo "PROCESSEDBY=${PROCESSEDBY}"

# If clustername is found (not empty)
if [ "${CLUSTERNAME}" != "" ]; then

case ${CHANGETYPE} in
  qtree-create)
# Check SVM at Cluster
    VSERVER=`SSHCMD ${CLUSTERNAME} "vserver show -operational-state running -vserver ${SVMNAME} -field vserver"|grep ${SVMNAME}|awk '{print $1}'`
# awk is NEEDED for selecting the vservername
    if [ "${VSERVER}" = "${SVMNAME}" ]; then
      VOLUME=`SSHCMD ${CLUSTERNAME} "volume show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume"|grep ${VOLNAME}|awk '{print $2}'`
      if [ "${VOLUME}" != "${VOLNAME}" ]; then
        echo "Volume ${VOLNAME} need to be created first ..."
      else
        QTREE=`SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -field qtree"|grep ${QTREENAME}|awk '{print $3}'`
        if [ "${QTREE}" != "${QTREENAME}" ]; then
# Qtree not found, so create
          echo "  Creating Qtree ${QTREENAME} at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..."|tee -a ${TMP}
          SSHCMD ${CLUSTERNAME} "qtree create -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -security-style ${QTREESTYLE}"
# Max 5 quota policies /SVM: Policynames by SVM-name
          SSHCMD ${CLUSTERNAME} "volume quota policy create -vserver ${SVMNAME} -policy-name ${SVMNAME}"

          SSHCMD ${CLUSTERNAME} "volume quota policy rule create -vserver ${SVMNAME} -volume ${VOLNAME} -policy-name ${SVMNAME} -target ${QTREENAME} -type tree -disk-limit ${QUOTASIZE}g"

          echo "- qtree-create: " >> ${TMP}
          SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -instance" >> ${TMP}
        else
          echo "  Qtree ${QTREENAME} already exists at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..." |tee ${ERR}|tee -a ${TMP} ${LOG}
        fi  # QTREE != QTREENAME
      fi  # vol != volname
    fi  # vserver = SVM
  ;;  # qtree-create

  qtree-delete)
#
# Check SVM at Cluster
    VSERVER=`SSHCMD ${CLUSTERNAME} "vserver show -operational-state running -vserver ${SVMNAME} -field vserver"|grep ${SVMNAME}|awk '{print $1}'`
# awk is NEEDED for selecting the vservername
    if [ "${VSERVER}" = "${SVMNAME}" ]; then
      VOLUME=`SSHCMD ${CLUSTERNAME} "volume show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume"|grep ${VOLNAME}|awk '{print $2}'`
      if [ "${VOLUME}" != "${VOLNAME}" ]; then
        echo "  Volume ${VOLNAME} not found ..."|tee ${ERR}|tee -a {LOG} ${TMP}
      else
        QTREE=`SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -field qtree"|grep ${QTREENAME}|awk '{print $3}'`
        if [ "${QTREE}" = "${QTREENAME}" ]; then
# Check for quota
        QUOTASTATE=`SSHCMD ${CLUSTERNAME} "quota show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume,state"|grep ${VOLNAME}|awk '{print $3}'`
        if [ "${QUOTASTATE}" = "on" ]; then
          echo "QUOTASTATE=${QUOTASTATE}"
          QUOTARULE=`SSHCMD ${CLUSTERNAME} "volume quota policy rule show -vserver ${SVMNAME} -volume ${VOLNAME} -target ${QTREE} -field policy-name"|grep ${QTREE}|awk '{print $2}'`
          QUOTATYPE=`SSHCMD ${CLUSTERNAME} "volume quota policy rule show -vserver ${SVMNAME} -volume ${VOLNAME} -target ${QTREE} -field policy-name"|grep ${QTREE}|awk '{print $4}'`
          if [ "${QUOTARULE}" != "" ] && [ "${QUOTATYPE}" != "" ]; then
            echo "  Deleting QUOTARULE=${QUOTARULE}, QUOTATYPE=${QUOTATYPE}"|tee -a ${LOG}
            SSHCMD ${CLUSTERNAME} "volume quota policy rule delete -vserver ${SVMNAME} -volume ${VOLNAME} -target ${QTREE} -policy-name ${QUOTARULE} -type ${QUOTATYPE}"
          fi  # [ "${QUOTARULE}" != "" ] && [ "${QUOTATYPE}" != "" ]
        fi


# Qtree found, so can be deleted
          echo "  Deleting ${QTREESIZE} Qtree ${QTREENAME} at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..."|tee -a ${TMP}
          echo "Last info: " >> ${TMP}
          SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -instance" >> ${TMP}

          SSHCMD ${CLUSTERNAME} "qtree delete -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -foreground true" |tee -a ${TMP}

        else
          echo "  Qtree ${QTREENAME} NOT found at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..." |tee ${ERR} |tee -a ${LOG} ${TMP}

        fi  # QTREE != QTREENAME
      fi  # vol != volname
    fi  # vserver = SVM
  ;;
  # qtree-delete

  qtree-modify)
# echo "  Qtree modify not implemented yet ..." |tee ${ERR}|tee -a ${LOG} ${TMP}
    VSERVER=`SSHCMD ${CLUSTERNAME} "vserver show -operational-state running -vserver ${SVMNAME} -field vserver"|grep ${SVMNAME}|awk '{print $1}'`
# awk is NEEDED for selecting the vservername
    if [ "${VSERVER}" = "${SVMNAME}" ]; then
      VOLUME=`SSHCMD ${CLUSTERNAME} "volume show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume"|grep ${VOLNAME}|awk '{print $2}'`
      if [ "${VOLUME}" != "${VOLNAME}" ]; then
        echo "  Volume ${VOLNAME} not found ..."|tee ${ERR}|tee -a {LOG} ${TMP}
      else
        QTREE=`SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -field qtree"|grep ${QTREENAME}|awk '{print $3}'`
        if [ "${QTREE}" = "${QTREENAME}" ]; then
#
# Check for quota
#
        QUOTASTATE=`SSHCMD ${CLUSTERNAME} "quota show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume,state"|grep ${VOLNAME}|awk '{print $3}'`
          if [ "${QUOTASTATE}" = "on" ]; then

            QUOTAPOLICYNAME=`SSHCMD ${CLUSTERNAME} "volume quota policy rule show -vserver ${SVMNAME} -volume ${VOLNAME} -target ${QTREE} -field policy-name"|grep ${QTREE}|awk '{print $2}'`
            QUOTATYPE=`SSHCMD ${CLUSTERNAME} "volume quota policy rule show -vserver ${SVMNAME} -volume ${VOLNAME} -target ${QTREE} -field policy-name"|grep ${QTREE}|awk '{print $4}'`

            if [ "${QUOTAPOLICYNAME}" != "" ] && [ "${QUOTATYPE}" != "" ]; then

              QUOTASIZEGB=`SSHCMD ${CLUSTERNAME} "set -unit GB; volume quota policy rule show -vserver ${SVMNAME} -volume ${VOLNAME} -target ${QTREE} -type ${QUOTATYPE} -field disk-limit"|grep ${QTREE}|awk '{print $7}'|sed 's/GB//g'`

echo "  VSERVER=${VSERVER} VOLUME=${VOLUME} QTREE=${QTREE} QUOTASTATE=${QUOTASTATE} QUOTAPOLICYNAME=${QUOTAPOLICYNAME} QUOTATYPE=${QUOTATYPE} QUOTASIZEGB=${QUOTASIZEGB} QUOTADELTASIZEGB=${QUOTADELTASIZEGB}" | tee -a ${TMP}

              if [ "QUOTADELTASIZEGB" != "" ] && [ "${QUOTASIZECHANGE}" = "I" ]; then
                 NEWQUOTASIZEGB=`expr ${QUOTASIZEGB} + ${QUOTADELTASIZEGB}`             
              fi
              if [ "QUOTADELTASIZEGB" != "" ] && [ "${QUOTASIZECHANGE}" = "D" ]; then
                 NEWQUOTASIZEGB=`expr ${QUOTASIZEGB} - ${QUOTADELTASIZEGB}`             
              fi
              if [ ${NEWQUOTASIZEGB} -le 0 ]; then
                NEWQUOTASIZEGB=1
                echo "NEWQUOTASIZEGB <= 0. So set to 1GB"|tee ${ERR}| tee -a ${TMP} {LOG}
              fi
echo "  NEWQUOTASIZEGB=${NEWQUOTASIZEGB}"| tee -a ${TMP}

              SSHCMD ${CLUSTERNAME} "volume quota policy rule modify -vserver ${SVMNAME} -volume ${VOLNAME} -policy-name ${QUOTAPOLICYNAME} -target ${QTREE} -type ${QUOTATYPE} -disk-limit ${NEWQUOTASIZEGB}GB " |tee -a ${TMP}

              SSHCMD ${CLUSTERNAME} "volume quota policy rule show -vserver ${SVMNAME} -volume ${VOLNAME} -policy-name ${QUOTAPOLICYNAME} -target ${QTREE} -type ${QUOTATYPE} -instance" | tee -a  ${TMP}



            fi  # [ "${QUOTARULE}" != "" ] && [ "${QUOTATYPE}" != "" ]
          fi  # "${QUOTASTATE}" = "on"
        fi  # "${QTREE}" = "${QTREENAME}"
      fi  # "${VOLUME}" != "${VOLNAME}"
    fi  # "${VSERVER}" = "${SVMNAME}"

  ;;  # qtree-modify


  qtree-show)
#    echo "  Qtree modify not implemented yet ..." |tee ${ERR}|tee -a ${LOG} ${TMP}

# Check SVM at Cluster
    VSERVER=`SSHCMD ${CLUSTERNAME} "vserver show -operational-state running -vserver ${SVMNAME} -field vserver"|grep ${SVMNAME}|awk '{print $1}'`
# awk is NEEDED for selecting the vservername
    if [ "${VSERVER}" = "${SVMNAME}" ]; then
      VOLUME=`SSHCMD ${CLUSTERNAME} "volume show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume"|grep ${VOLNAME}|awk '{print $2}'`
      if [ "${VOLUME}" != "${VOLNAME}" ]; then
        echo "  Volume ${VOLNAME} not found ..."|tee ${ERR}|tee -a {LOG} ${TMP}
      else
        if [ "${QTREENAME}" = "*" ]; then
          SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -instance" >> ${TMP}
        else
          QTREE=`SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME}" -field qtree"|grep ${QTREENAME}|awk '{print $3}'`
          if [ "${QTREE}" = "${QTREENAME}" ]; then
            SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -instance" >> ${TMP}
          else
            echo "  Qtree ${CLUSTERNAME}/${SVMNAME}:${VOLNAME}/${QTREENAME} not found ..."|tee ${ERR}|tee -a {LOG} ${TMP}
         fi  # "${QTREENAME}" = "*"
        fi  # "${QTREE}" = "${QTREENAME}"
      fi  # "${VOLUME}" != "${VOLNAME}"
    fi  # "${VSERVER}" = "${SVMNAME}"
  ;; # qtree-show

  *)
    echo "  Change ${CHANGETYPE} NOT known." |tee ${ERR}|tee -a ${LOG} ${TMP}
  ;;  # *
esac


else
  echo "  No clustername found. Done nothing." |tee ${ERR}|tee -a ${LOG} ${TMP}
fi  # CLUSTERNAME


# Mailing part
if [ -s ${TMP} ]; then  # If size > 0
  cp ${TMP} ${ASC}
  if  [ -s ${ERR} ]; then
    echo "`date` Change ${CHANGETYPE} finished with ERROR (`cat ${ERR}`). See attachement for more info." | mailx -a ${ASC} -s "FSOD change with ERRORS [${PGM} v${VER}]" ${MAILADDRESS}
  else
    echo "`date` Change ${CHANGETYPE} finished OK. See attachement for more info." | mailx -a ${ASC} -s "FSOD change = OK [${PGM} v${VER}]" ${MAILADDRESS}
  fi  #  [ -s ${ERR} ]
  echo "  Send mail about ${CHANGETYPE} to ${MAILADDRESS}" |tee -a ${LOG}
  rm ${ASC}
fi  # -s ${TMP}


echo "  Removing RCfile ${RCFILE}" |tee -a ${LOG}
rm ${RCFILE}

rm ${TMP} ${ERR}
echo "`date` ${PGM} v${VER} finished." >> ${LOG}
exit 0