#!/bin/bash
# File	: proc_fsod-change.sh
# By	: Maarten.deBoer@Atos.net, 180207
# Subject	: Script to process FSOD-changes
#set -x
PGM=`basename $0|cut -d\. -f1`
VER="0.1"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
ASC="/tmp/${PGM}.asc"
ERR="/tmp/${PGM}.err"

RCFILE="/tmp/fsod-change.rc"
CLUSTER_VSERVER_FILE="/home/aodfm01a/export/fsod-clusters-vservers.csv"

FSOD_MGTSRV="aoxnlfsd01"
CLUSTER="nlnaf100"


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG}
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

# MAIN
echo "`date` ${PGM} v${VER} started." >> ${LOG}
touch ${TMP} ${ERR}

if [ -f ${RCFILE} ]; then
  . ${RCFILE}
else
  echo "  NO RCfile (${RCFILE}). So exiting ..."|tee -a ${LOG}
  exit 4
fi

echo "  With DATETIMESTAMP=${DATETIMESTAMP}, CHANGETYPE=${CHANGETYPE},CHANGENUMBER=${CHANGENUMBER},CLUSTERNAME=${CLUSTERNAME},SVMNAME=${SVMNAME},MAILADDRESS=${MAILADDRESS}"|tee -a ${LOG}

if [ "${SVMNAME}" != "" ] && [ "${CLUSTERNAME}" = "" ]; then
  CLUSTERNAME=`grep ${SVMNAME} ${CLUSTER_VSERVER_FILE}|cut -d\; -f1`
fi  # CLUSTERNAME
echo "CLUSTERNAME=${CLUSTERNAME}"
echo "SVMNAME=${SVMNAME}"
echo "CHANGENUMBER=${CHANGENUMBER}"
echo "CHANGETYPE=${CHANGETYPE}"
echo "VOLNAME=${VOLNAME}"
echo "QTREENAME=${QTREENAME}"
echo "QTREESTYLE=${QTREESTYLE}"
echo "QTREESIZE=${QTREESIZE}"
echo "MAILADDRESS=${MAILADDRESS}"
echo "DATETIMESTAMP=${DATETIMESTAMP}"
echo "PROCESSEDBY=${PROCESSEDBY}"

# If clustername is found (not empty)
if [ "${CLUSTERNAME}" != "" ]; then
#
# qtree-create
#
  if [ "${CHANGETYPE}" = "qtree-create" ]; then

# Check SVM at Cluster
    VSERVER=`SSHCMD ${CLUSTERNAME} "vserver show -operational-state running -vserver ${SVMNAME} -field vserver"|grep ${SVMNAME}|awk '{print $1}'`
# awk is NEEDED for selecting the vservername
    if [ "${VSERVER}" = "${SVMNAME}" ]; then
      VOLUME=`SSHCMD ${CLUSTERNAME} "volume show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume"|grep ${VOLNAME}|awk '{print $2}'`
      if [ "${VOLUME}" != "${VOLNAME}" ]; then
        echo "Volume ${VOLNAME} need to be created first ..."
      else
        QTREE=`SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -field qtree"|grep ${QTREENAME}|awk '{print $3}'`
        if [ "${QTREE}" != "${QTREENAME}" ]; then
# Qtree not found, so create
          echo "  Creating Qtree ${QTREENAME} at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..."|tee -a ${TMP}
          SSHCMD ${CLUSTERNAME} "qtree create -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -security-style ${QTREESTYLE}"

          echo "# `date` ${PGM} v${VER} started." >> ${TMP}
          echo "- qtree-create: " >> ${TMP}
          SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -instance" >> ${TMP}
        else
          echo "  Qtree ${QTREENAME} already exists at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..." |tee -a ${TMP} ${ERR} ${LOG}
        fi  # QTREE != QTREENAME
      fi  # vol != volname
    fi  # vserver = SVM
  fi  # qtree-create
#
# qtree-delete
#
  if [ "${CHANGETYPE}" = "qtree-delete" ]; then
# Check SVM at Cluster
    VSERVER=`SSHCMD ${CLUSTERNAME} "vserver show -operational-state running -vserver ${SVMNAME} -field vserver"|grep ${SVMNAME}|awk '{print $1}'`
# awk is NEEDED for selecting the vservername
    if [ "${VSERVER}" = "${SVMNAME}" ]; then
      VOLUME=`SSHCMD ${CLUSTERNAME} "volume show -vserver ${SVMNAME} -volume ${VOLNAME} -field volume"|grep ${VOLNAME}|awk '{print $2}'`
      if [ "${VOLUME}" != "${VOLNAME}" ]; then
        echo "  Volume ${VOLNAME} not found ..."|tee -a ${ERR} {LOG} ${TMP}
      else
        QTREE=`SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -field qtree"|grep ${QTREENAME}|awk '{print $3}'`
        if [ "${QTREE}" = "${QTREENAME}" ]; then
# Qtree found, so can be deleted
          echo "  Deleting ${QTREESIZE} Qtree ${QTREENAME} at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..."|tee -a ${TMP}
          echo "Last info: " >> ${TMP}
          SSHCMD ${CLUSTERNAME} "qtree show -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME} -instance" >> ${TMP}

          SSHCMD ${CLUSTERNAME} "qtree delete -vserver ${SVMNAME} -volume ${VOLNAME} -qtree ${QTREENAME}"

        else
          echo "  Qtree ${QTREENAME} NOT found at ${VOLNAME} at ${SVMNAME} at ${CLUSTERNAME} ..." |tee -a ${LOG} ${ERR} ${TMP}

        fi  # QTREE != QTREENAME
      fi  # vol != volname
    fi  # vserver = SVM
  fi  # qtree-delete
#
# qtree-modify
#
  if [ "${CHANGETYPE}" = "qtree-modify" ]; then
    echo "  Qtree modify not implemented yet ..." |tee -a ${LOG} ${ERR} ${TMP}
  fi  # qtree-modify


else
  echo "  No clustername found. Done nothing." |tee -a ${ERR} ${LOG} ${TMP}
fi  # CLUSTERNAME

if [ -s ${TMP} ]; then  # If size > 0
  cp ${TMP} ${ASC}
  if  [ -s ${ERR} ]; then
    echo "`date` finished with ERRORs. See attachement." | mailx -a ${ASC} -s "FSOD change with ERRORS [${PGM} v${VER}]" ${MAILADDRESS}
  else
    echo "`date` finished OK. See attachement." | mailx -a ${ASC} -s "FSOD change = OK [${PGM} v${VER}]" ${MAILADDRESS}
  fi  #  [ -s ${ERR} ]
  echo "  Send mail about ${CHANGETYPE} to ${MAILADDRESS}" |tee -a ${LOG}
  rm ${ASC}
fi  # -s ${TMP}


echo "  Removing RCfile ${RCFILE}" |tee -a ${LOG}
rm ${RCFILE}

rm ${TMP} ${ERR}
echo "`date` ${PGM} v${VER} finished." >> ${LOG}
exit 0