#!/bin/sh
# File	: cdot_rpt_lif-qtree.sh
# By	: Maarten.deBoer@Atos.net, 211209
# Subject	: Script to report LIF's and (part of) Qtrees (for SAP)
#(0.2),211216	: Add "orphaned-LIF" (LIF without related Qtree), ORPH_LIFS
#(0.3),220503	: Add create_migrate_script (CRE_IMS)
#(0.4),220518	: Mod. new-node, Add SIDFILTER
#(0.5),220518	: Add PING_ADDR
#(0.6),220926	: Added REMARK (Same / NOT same)
PGM="`basename $0|cut -d\. -f1`"
VER="0.6"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
RCFILE="${HOME}/etc/${PGM}.rc"
SSH="/usr/bin/ssh -n"
MAILTO="maarten.deboer@atos.net"
CLUSTERS="${HOME}/etc/clusters"
CSV="/tmp/${PGM}.csv"
MAIL=""
HOSTNAME=`hostname |cut -d\. -f2`
CFILTER="[?]*"
NFILTER="*"
SFILTER="nlnafs"
ORPH_LIFS=""
CRE_IMS=""
CRE_VMS=""
LIFMOVE_FILE="${HOME}/data/${PGM}_lif-move.sh"
VOLMOVE_FILE="${HOME}/data/${PGM}_vol-move.sh"
NODES_EXCL="n02a|n09a|n10a"  # By excluding the aggr-(part)names
SIDFILTER="[?]*"
DEST_AGGR="n11a3801"
PING_ADDR="192.168.9.195"
REMARK=""


# sshcmd.cunf
#--------------------------------------------------------------------#
# File	: sshcmd.func
# VER="0.1", <node>-04
# VER="0.2", <node>-06
# VER="0.3", <node>-10
# VER="0.4", nodes -01, -05..13
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.tmperr"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to the host port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                sleep 1
                /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
                EC=${?}
                grep 'Connection refused' ${TMPERR}
                EC2=${?}
                if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                  sleep 1
                  /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
                  EC=${?}
                  grep 'Connection refused' ${TMPERR}
                  EC2=${?}
                  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                    sleep 1
                    /usr/bin/ssh -n ${1}-12 "${2}" 2> ${TMPERR}
                    EC=${?}
                    grep 'Connection refused' ${TMPERR}
                    EC2=${?}
                    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                      sleep 1
                      /usr/bin/ssh -n ${1}-13 "${2}" 2> ${TMPERR}
                      EC=${?}
                      grep 'Connection refused' ${TMPERR}
                      EC2=${?}
                      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                        echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01,-05 .. -13 failed too."|tee -a ${LOG} 
                      fi  # -10
                    fi  # -09
                  fi  # -08
                fi  # -07
              fi  # -06
            fi  # -05
          fi  # -04
        fi  # -03
      fi  # -02
    fi  # -01
  fi  # 
  rm ${TMPERR}
  return ${EC}
}

USAGE()
{
  echo "Usage: $PGM [options] "
  echo "  Version: ${VER}"
  echo "  options       :"
  echo "    -c          : Cluster filter (${CFILTER})"
  echo "    -n          : Node filter (${NFILTER})"
  echo "    -s          : Svm filter (${SFILTER})"
  echo "    -e|--etc    : Etc/clusters-file (${CLUSTERS})"
  echo "    -h          : this help"
  echo "    -m|--mail   : send Mail"
  echo "    -o          : report Orphaned lifs (LIF without related Qtree)"
  echo "    -V          : Version"
  echo "    -x          : set -x"
  echo "    --mailto    : change MAILTO address & do send mail (${MAILTO})"
  echo "    --ims       : create Interface Migrate Script (LIF-moves)"
  echo "    --vms       : create Volume Move Script (VOL-moves)"
  echo "    --sid       : SID-filter (${SID_FILTER})"
  echo "    --d-aggr    : Destination AGGRegate (${DEST_AGGR})"
  echo "    --ping-addr : PING ADDR (${PING_ADDR}) for in LIF-move-script (${LIFMOVE_FILE})"
}
if [ -f ${RCFILE} ]; then
  echo "  Reading RCFILE (${RCFILE})"|tee -a ${LOG}
  . ${RCFILE}
fi  # RCFILE
# Check options
while [ $# -gt 0 ]
  do
  case ${1} in
    -c) CFILTER=${2}; shift ;;
    -e | --etc) CLUSTERS=${2}; shift ;;
    -h | --help) USAGE; exit 1 ;;
    --ims) CRE_IMS=1 ;;
    --vms) CRE_VMS=1 ;;
    -m | --mail) MAIL=1 ;;
    --mailto) MAILTO=$2; MAIL=1; shift ;;
    -n) NFILTER=${2}; shift ;;
    -s) SFILTER=${2}; shift ;;
    --sid) SIDFILTER=${2}; shift ;;
    --d-aggr) DEST_AGGR=${2}; shift ;;
    --ping-addr) PING_ADDR=${2}; shift ;;
    -o) ORPH_LIFS=1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG} ${TMP}.m
echo "CLUSTERS=${CLUSTERS}"|tee -a ${TMP}.m
echo "CFILTER=${CFILTER}"|tee -a ${TMP}.m
echo "NFILTER=${NFILTER}"|tee -a ${TMP}.m
echo "SFILTER=${SFILTER}"|tee -a ${TMP}.m
echo "SIDFILTER=${SIDFILTER}"|tee -a ${TMP}.m
echo "MAIL=${MAIL}"|tee -a ${TMP}.m
echo "MAILTO=${MAILTO}"|tee -a ${TMP}.m
echo "ORPH_LIFS=${ORPH_LIFS}"|tee -a ${TMP}.m
echo "CRE_IMS=${CRE_IMS}"|tee -a ${TMP}.m
echo "LIFMOVE_FILE=${LIFMOVE_FILE}"|tee -a ${TMP}.m
echo "CRE_VMS=${CRE_VMS}"|tee -a ${TMP}.m
echo "VOLMOVE_FILE=${VOLMOVE_FILE}"|tee -a ${TMP}.m
echo "DEST_AGGR=${DEST_AGGR}"|tee -a ${TMP}.m
echo "PING_ADDR=${PING_ADDR}"|tee -a ${TMP}.m
sleep 1

echo "# cluster;vserver;home-node;curr-node;lif;(sid) lif;qtree;volume;aggregate;aggr-node;new-node;remark;" > ${TMP}

for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"`
do
  echo "  ${CLUSTER}:"
  SSHCMD ${CLUSTER} "vserver show -field vserver"|grep ${SFILTER}|while read VSERVER REST
  do
    echo "    |${VSERVER}|"
    SSHCMD ${CLUSTER} "set -showseparator \";\" ; network interface show -vserver ${VSERVER} -curr-node ${NFILTER} -field lif,curr-node,home-node"|grep ${VSERVER}|grep "${SIDFILTER}"|while read LINE
    do
      if [ "${LINE}" != "" ]; then
#     nlnafsakz06;nlnafsakz06;akz06-lt0;akz06-lt0;nlnaf100-03;
#        echo "    ${LINE}"
        VSERVER=`echo ${LINE}|awk -F\; '{print $1}'`
        LIF=`echo ${LINE}|awk -F\; '{print $3}'`
        SID_LIF=`echo ${LINE}|awk -F\; '{print $3}'|cut -d\- -f2|cut -d\_ -f2`
        HOME_NODE=`echo ${LINE}|awk -F\; '{print $5}'`
        CURR_NODE=`echo ${LINE}|awk -F\; '{print $6}'`

        SSHCMD ${CLUSTER} "set -showseparator \";\" ;qtree show -vserver ${VSERVER} -qtree *${SID_LIF} -field qtree,volume"| grep ${VSERVER}|while read LINE2
        do
#      nlnafsakz06;akz06_vol01098;sap_ers_hqp;
#          echo "      ${LINE2}"
          VOLUME=`echo ${LINE2}|awk -F\; '{print $2}'`
          QTREE=`echo ${LINE2}|awk -F\; '{print $3}'`
          SSHCMD ${CLUSTER} "set -showseparator \";\" ;volume show -vserver ${VSERVER} -volume ${VOLUME} -field volume,aggregate,node"| grep ${VSERVER}|while read LINE3
          do
#            echo "      ${LINE3}"
#      nlnafsakz06;akz06_vol01102;n07a3801;nlnaf100-07;

            AGGR=`echo ${LINE3}|awk -F\; '{print $3}'`
            AGGR_NODE=`echo ${LINE3}|awk -F\; '{print $4}'`

            NEW_NODE="${AGGR_NODE}"
# Check new-node
            REMARK=""
            if [ "${NEW_NODE}" = "${HOME_NODE}" ]; then
              REMARK=" New-node=Home-node"
            fi
            if [ "${NEW_NODE}" = "${CURR_NODE}" ]; then
              REMARK="${REMARK} New-node=Curr-node"
            fi
            if [ "${REMARK}" = "" ]; then
              REMARK="NOT same"
            fi

# Put into TMP(CSV)-file
            echo "${CLUSTER};${VSERVER};${HOME_NODE};${CURR_NODE};${LIF};${SID_LIF};${QTREE};${VOLUME};${AGGR};${AGGR_NODE};${NEW_NODE};${REMARK};"|tee -a ${TMP}

          done  # LINE3

        done  # qtree | LINE2

# Check/search for Orphaned-LIFs (LIF without related Qtree)
        if [ ${ORPH_LIFS} ]; then
          echo "      Check for Orphaned-LIFs" |tee -a ${LOG}
          ANSW=`SSHCMD ${CLUSTER} "set -showseparator \";\" ;qtree show -vserver ${VSERVER} -qtree *${SID_LIF} -field qtree,volume"| grep "no entries"`
          if [ "${ANSW}" != "" ] && [ "${LIF}" != "${SID_LIF}" ]; then
            # When <> "" then no entries. So, Orphaned found
            # And LIF <> SID_LIF (otherwise "main" LIF)
            echo "${CLUSTER};${VSERVER};${CURR_NODE};${LIF};${SID_LIF};;;;;;;No Qtrees found. Possible Orphaned LIF (LIF without Qtrees)"|tee -a ${TMP}
          fi  # ANSW != ""
        fi  # ORPH_LIFS

      fi  # != ""
    done  # network | LINE
  done  # vserver
done  # for CLUSTER


if [ ${CRE_IMS} ]; then
  touch ${TMP}.LIF
  echo "# Interface Migrate Script " > ${LIFMOVE_FILE}
  echo "# By ${PGM} v${VER} at `hostname` at `date`" > ${LIFMOVE_FILE}
# Create Interface Migrate Script from ${TMP}
# echo "${CLUSTER};${VSERVER};${CURR_NODE};${LIF};${SID_LIF};${QTREE};${VOLUME};${AGGR};${AGGR_NODE};"|tee -a ${TMP}
  cat ${TMP}|grep -v ^#|grep "${SIDFILTER}"| while read LINE
  do
    CLUSTER=`echo ${LINE}|awk -F\; '{print $1}'`
#    VSERVER=`echo ${LINE}|awk -F\; '{print $2}'`
    LIF=`echo ${LINE}|awk -F\; '{print $5}'`
#    AGGR_NODE=`echo ${LINE}|awk -F\; '{print $9}'`
#    echo "${CLUSTER};${VSERVER};${LIF};${AGGR_NODE};" >> ${TMP}.2
    echo "${CLUSTER};${LIF};" >> ${TMP}.LIF
  done  # LINE

#  cat ${TMP}.LIF|grep -v ^#|sort -u|egrep -v "${NODES_EXCL}"|while read LINE2
  cat ${TMP}.LIF|grep -v ^#|sort -u|while read CLUSTER_LIF
  do
    echo ${CLUSTER_LIF}
    CLUSTER=`echo ${CLUSTER_LIF}|awk -F\; '{print $1}'`
    LIF=`echo ${CLUSTER_LIF}|awk -F\; '{print $2}'`
# Select the Cluster+LIF WITHOUT EXCLuded  (SnapVault) nodes
    LINE2=`cat ${TMP}|grep "${CLUSTER}"|grep "${LIF}"|egrep -v "${NODES_EXCL}"|grep "${SIDFILTER}"|awk -F\; '{print $1";"$2";"$5";"$10";"}'|sort -u`
    echo "LINE2=${LINE2}"

    CLUSTER=`echo ${LINE2}|awk -F\; '{print $1}'`
    VSERVER=`echo ${LINE2}|awk -F\; '{print $2}'`
    LIF=`echo ${LINE2}|awk -F\; '{print $3}'`
    AGGR_NODE=`echo ${LINE2}|awk -F\; '{print $4}'`

# Commands for LIF move & show
    echo "ssh ${CLUSTER} \"network interface show -vserver ${VSERVER} -lif ${LIF} -field lif,curr-node,home-node\" "  >> ${LIFMOVE_FILE}
    echo "ssh ${CLUSTER} \"network interface modify -vserver ${VSERVER} -lif ${LIF} -home-node ${AGGR_NODE}\" "  >> ${LIFMOVE_FILE}
    echo "ssh ${CLUSTER} \"network interface migrate -vserver ${VSERVER} -lif ${LIF} -destination-node ${AGGR_NODE}\" "  >> ${LIFMOVE_FILE}
    echo "ssh ${CLUSTER} \"network interface show -vserver ${VSERVER} -lif ${LIF} -field lif,curr-node,home-node\" "  >> ${LIFMOVE_FILE}
    echo "ssh ${CLUSTER} \"ping -vserver ${VSERVER} -lif ${LIF} -destination ${PING_ADDR} \" "  >> ${LIFMOVE_FILE}
    echo "# -------------------------------------------" >> ${LIFMOVE_FILE}

  done   # CLUSTER_LIF

  echo ""|tee -a ${TMP}.m
  echo "  Created Interface Migration Script (lif-move) ${LIFMOVE_FILE}"|tee -a ${LOG} ${TMP}.m
  rm ${TMP}.LIF
fi  # CRE_IMS

if [ ${CRE_VMS} ]; then
  echo "# Volume Move Script " > ${VOLMOVE_FILE}
  echo "# By ${PGM} v${VER} at `hostname` at `date`" > ${VOLMOVE_FILE}

# Select the Cluster+LIF WITHOUT EXCLuded  (SnapVault) nodes
  cp /dev/null ${TMP}.vms
  cat ${TMP}|grep -v ^#|grep "${CFILTER}"|grep "${SFILTER}"|egrep -v "${NODES_EXCL}"|grep "${SIDFILTER}"| while read LINE
  do
#    echo "  LINE=${LINE}"
    CLUSTER=`echo ${LINE}|awk -F\; '{print $1}'`
    VSERVER=`echo ${LINE}|awk -F\; '{print $2}'`
    VOLUME=`echo ${LINE}|awk -F\; '{print $8}'`
    echo "${CLUSTER};${VSERVER};${VOLUME};" >> ${TMP}.vms
# Creating a TeMP-file for sorting only 1 volume
  done  # cat ${TMP}

  cat ${TMP}.vms|sort -u| while read LINE
  do
    if [ "${LINE}" != "" ]; then
      echo "  LINE=${LINE}"
      CLUSTER=`echo ${LINE}|awk -F\; '{print $1}'`
      VSERVER=`echo ${LINE}|awk -F\; '{print $2}'`
      VOLUME=`echo ${LINE}|awk -F\; '{print $3}'`
    fi  # ${LINE}" != ""
# Commands for volume move & show
    echo "ssh ${CLUSTER} \"volume move start -vserver ${VSERVER} -volume ${VOLUME}  -destination-aggregate ${DEST_AGGR} \" "  >> ${VOLMOVE_FILE}
    echo "ssh ${CLUSTER} \"volume move show -vserver ${VSERVER}\" "  >> ${VOLMOVE_FILE}

  done  # cat ${TMP}.vms
  echo "echo \"tail -2 ${VOLMOVE_FILE} \" | head -1 "  >> ${VOLMOVE_FILE}

  rm ${TMP}.vms

  echo ""|tee -a ${TMP}.m
  echo "  Created Volume Move Script (vol-move) ${VOLMOVE_FILE}"|tee -a ${LOG} ${TMP}.m
fi  # CRE_VMS


if [ ${MAIL} ]; then
  cp ${TMP} ${CSV}
  cat ${TMP}.m|mailx -a ${CSV} -s ":${HOSTNAME}: LIF and Qtree report [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  rm ${CSV} ${TMP}.m 
fi  # MAIL

rm ${TMP}
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0