#!/bin/sh
# File	: cdot_snapmirror-volume_release-delete.sh
# By	: Maarten.deBoer@Atos.net, 211115
# Subject	: Script to release SnapMirror (if any) from offline volumes. And delete (if wanted, choice) the offline volume
#(0.2),240616	: From cdot_snapmirror-release_volume-delete.sh
#(0.3),240624	: Added SSHCMD
PGM=`basename $0|cut -d\ -f1`
VER="0.3"
LOG="${HOME}/log/${PGM}.log"
TMP="/tmp/${PGM}.$$"
CLUSTERS="${HOME}/etc/clusters"
# Filters
CFILTER="nlnaf10[01]"
SFILTER="[*]?"
VFILTER="[*]?"
SSH="/usr/bin/ssh -n"
#REMOVE_SM=""
REMOVE_SM=1
#DELETE_VOL=""
DELETE_VOL=1

# File	: sshcmd.func
# VER="0.1", <node>-04
# VER="0.2", <node>-06
# VER="0.3", <node>-10
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.tmperr"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                sleep 1
                /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
                EC=${?}
                grep 'Connection refused' ${TMPERR}
                EC2=${?}
                if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                  sleep 1
                  /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
                  EC=${?}
                  grep 'Connection refused' ${TMPERR}
                  EC2=${?}
                  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                    sleep 1
                    /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
                    EC=${?}
                    grep 'Connection refused' ${TMPERR}
                    EC2=${?}
                    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                      sleep 1
                      /usr/bin/ssh -n ${1}-12 "${2}" 2> ${TMPERR}
                      EC=${?}
                      grep 'Connection refused' ${TMPERR}
                      EC2=${?}
                      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                        echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 .. -10 failed too."|tee -a ${LOG} 
                      fi 
                    fi
                  fi 
                fi 
              fi 
            fi  
          fi  
        fi  
      fi  
    fi  
  fi  # 
  rm ${TMPERR}
}


if [ "${1}" != "" ]; then
  CFILTER="${1}"
fi  # $1
if [ "${2}" != "" ]; then
  SFILTER="${2}"
fi  # $2
if [ "${3}" != "" ]; then
  VFILTER="${3}"
fi  # $2

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "  CFILTER(1)=${CFILTER}"
echo "  SFILTER(2)=${SFILTER}"
echo "  VFILTER(3)=${VFILTER}"
echo "  REMOVE_SM=${REMOVE_SM}"
echo "  DELETE_VOL=${DELETE_VOL}"
sleep 1

# Check offline volumes 1st
cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"|while read CLUSTER
do
  echo "  Checking offline volumes at ${CLUSTER} ..."
  SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume show -state offline -field vserver,volume,state"|grep offline|grep -E "${SFILTER}"|grep -E "${VFILTER}"|while read LINE
  do
    echo ""
    VSERVER=`echo ${LINE}|awk -F\; '{print $1}'`
    VOLUME=`echo ${LINE}|awk -F\; '{print $2}'`
    echo "  VSERVER=${VSERVER} VOLUME=${VOLUME}"

    if [ "${VSERVER}" != "" ] && [ "${VOLUME}" != "" ]; then
# Check if SnapMirror relation.
# If so, then release (not empty)
      SNAPMIRROR_SRC=`SSHCMD ${CLUSTER} "snapmirror show -source-volume ${VOLUME} -field vserver,source-path,destination-path"|grep "${VOLUME}"`
      echo "  SNAPMIRROR_SRC=${SNAPMIRROR_SRC}|"
      if [ "${SNAPMIRROR_SRC}" != "" ]; then
        SM_SRC_PATH=`echo ${SNAPMIRROR_SRC}|awk '{print $1}'`
        SM_DEST_PATH=`echo ${SNAPMIRROR_SRC}|awk '{print $4}'`
        echo "    Removing SnapMirror ${SNAPMIRROR_SRC} (${SM_SRC_PATH} ${SM_DEST_PATH})"|tee -a ${LOG}
        SSHCMD ${CLUSTER} "volume online -vserver ${VSERVER} -volume ${VOLUME}"
        SSHCMD ${CLUSTER} "snapmirror delete -source-path ${SM_SRC_PATH} -destination-path ${SM_DEST_PATH}"
        SSHCMD ${CLUSTER} "volume offline -vserver ${VSERVER} -volume ${VOLUME}"
      fi  # SNAPMIRROR_SRC

      SNAPMIRROR_DEST=`SSHCMD ${CLUSTER} "snapmirror show -destination-volume ${VOLUME} -field source-path,destination-path"|grep "${VOLUME}"`
      echo "  SNAPMIRROR_DEST=${SNAPMIRROR_DEST}|"
      if [ "${SNAPMIRROR_DEST}" != "" ]; then
        SM_SRC_PATH=`echo ${SNAPMIRROR_DEST}|awk '{print $1}'`
        SM_DEST_PATH=`echo ${SNAPMIRROR_DEST}|awk '{print $2}'`
        echo "    Removing SnapMirror ${SNAPMIRROR_DEST} (${SM_SRC_PATH} ${SM_DEST_PATH})"|tee -a ${LOG}
        SSHCMD ${CLUSTER} "volume online -vserver ${VSERVER} -volume ${VOLUME}"
        SSHCMD ${CLUSTER} "snapmirror delete -source-path ${SM_SRC_PATH} -destination-path ${SM_DEST_PATH}"
        SSHCMD ${CLUSTER} "volume offline -vserver ${VSERVER} -volume ${VOLUME}"
      fi  # SNAPMIRROR_SRC



# snapmirror list-destinations
      SNAPMIRROR_SRC=`SSHCMD ${CLUSTER} "snapmirror list-destinations -source-vserver ${VSERVER} -source-volume ${VOLUME} -field source-vserver,source-volume"|grep "${VOLUME}"`
      echo "  SNAPMIRROR_SRC=${SNAPMIRROR_SRC}"
      if [ "${SNAPMIRROR_SRC}" != "" ]; then
        SM_SRC_PATH=`echo ${SNAPMIRROR_SRC}|awk '{print $1}'`
        SM_DEST_PATH=`echo ${SNAPMIRROR_SRC}|awk '{print $4}'`
        echo "    Releasing ${SNAPMIRROR_SRC} (${SM_SRC_PATH} ${SM_DEST_PATH})"|tee -a ${LOG}
        SSHCMD ${CLUSTER} "volume online -vserver ${VSERVER} -volume ${VOLUME}"
        SSHCMD ${CLUSTER} "snapmirror release -source-path ${SM_SRC_PATH} -destination-path ${SM_DEST_PATH}"
        SSHCMD ${CLUSTER} "volume offline -vserver ${VSERVER} -volume ${VOLUME}"

      fi  # SNAPMIRROR_SRC

      SSHCMD ${CLUSTER} "snapmirror list-destinations -destination-vserver ${VSERVER} -field source-vserver,source-volume,destination-vserver,destination-volume,type"|grep "${VOLUME}"| while read SNAPMIRROR_DEST
      do
        echo "    SNAPMIRROR_DEST=${SNAPMIRROR_DEST}|"

        if [ "${SNAPMIRROR_DEST}" != "" ]; then
          SM_SOURCE_PATH=`echo ${SNAPMIRROR_DEST}|awk -F\; '{print $1}'`
          SM_DEST_PATH=`echo ${SNAPMIRROR_DEST}|awk -F\; '{print $4}'`
          SM_TYPE=`echo ${SNAPMIRROR_DEST}|awk -F\; '{print $7}'`
          echo "    DEST: SM_SOURCE=${SM_SOURCE_PATH} SM_TYPE=${SM_TYPE} SM_DEST=${SM_DEST_PATH}"

          if [ ${REMOVE_SM}  ]; then
            echo "    Releasing SnapMirror ${SM_SOURCE} ${SM_TYPE} ${SM_DEST}"|tee -a ${LOG}
            SSHCMD ${CLUSTER} "volume online -vserver ${VSERVER} -volume ${VOLUME}"
            SSHCMD ${CLUSTER} "snapmirror release -destination-path ${SM_DEST_PATH}"
            SSHCMD ${CLUSTER} "volume offline -vserver ${VSERVER} -volume ${VOLUME}"
          fi #  REMOVE_SM
        fi  # SNAPMIRROR_DEST
      done  # SNAPMIRROR_DEST

      if [ ${DELETE_VOL} ]; then
        echo "  Deleting ${VSERVER}:${VOLUME}"|tee -a ${LOG}
        sleep 2
        SSHCMD ${CLUSTER} "volume delete -vserver ${VSERVER} -volume ${VOLUME}"
      fi  # DELETE_VOL
    fi  # <> ""

  done  # LINE
done  # CLUSTER

echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0