#!/bin/sh
# File	: cdot_init_vol_move.sh
# By	: Maarten.deBoer@Atos.net, 220808 & Andre.Hilgersom@Atos.net (2022)
# Subject	: Script to init / start volume moves
# Cmd: "vol move start -vserver $VSERV $VOL -desti ${3}"
#(0.2),220808	: Added USAGE, AFILTER
#(0.3),220809	: Added Aggr % check
#(0.4),220809	: Clever request info from Clusters
#(0.5),220817	: Check VAR's
#(0.6),221208	: Some mod's
#(0.7),240415	: Combien _init_ & _rpt_
PGM=`basename $0|cut -d\. -f1` 
VER="0.7"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
CSV="${HOME}/data/${PGM}.csv"
ASC="/tmp/${PGM}.asc"
SSH="/bin/ssh -n"
HOSTNAME=`hostname |cut -d\. -f1`
CLUSTERS="${HOME}/etc/clusters"
INIT=""
RPT=""

AFILTER="[?]*"
CFILTER="[?]*"
VFILTER="[?]*"
MAX_MOVES=3
MAX_AGGR_PERC=80
MAILTO="maarten.deboer@atos.net"
MAIL=""


USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options          :"
  echo "    -a|--aggr      : filter for AGGRname (${AFILTER})"
  echo "    -c|--cluster   : filter for Clustername (${CFILTER})"
  echo "    -v|--volume    : filter for Volume (${VFILTER})"
  echo "    -m|--max       : Max(${MAX_MOVES}) numbers of moves /cluster"
  echo "    -p|--perc      : Max(${MAX_AGGR_PERC}) aggr percentage before next move"
  echo "    -i|--init      : Initialize volume moves"
  echo "    -r|--rpt       : Report volume moves"
  echo "    -h|--help      : this Help"
  echo "    --mailto       : change MAILTO address (${MAILTO})"
  echo "    -V             : show Version"
  echo "    -x             : set -x"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -i | --init) INIT=1 ;;
    -r | --rpt) RPT=1 ;;
    -a | --aggr) AFILTER="${2}"; shift ;;
    -c | --cluster) CFILTER="${2}"; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -m | --max) MAX_MOVES=${2}; shift ;;
    -p | --perc) MAX_AGGR_PERC=${2}; shift ;;
    -v | --volume) VFILTER="${2}"; shift ;;
    --mailto) MAILTO="${2}"; MAIL=1 ; shift ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;;
  esac
    shift
done  # case

echo "`date`: ${PGM} v${VER} started."|tee -a ${LOG}
echo "  ASC=${ASC}"
echo "  CSV=${CSV}"
echo "  CLUSTERS=${CLUSTERS}"
echo "  INIT=${INIT}"
echo "  RPT=${RPT}"
echo "  AFILTER=${AFILTER}"
echo "  CFILTER=${CFILTER}"
echo "  VFILTER=${VFILTER}"
echo "  MAIL=${MAIL}"
echo "  MAILTO=${MAILTO}"
echo "  MAX_MOVES=${MAX_MOVES}"
echo "  MAX_AGGR_PERC=${MAX_AGGR_PERC}"
sleep 1

if [ ${INIT} ]; then
  echo "  Initializing ..."
  if [ ! -f ${CSV} ]; then
    echo "  NO CSV(data)-file found. Exiting ..."|tee -a ${LOG}
    exit 4
  fi
  cat ${CSV}|grep -v ^#|grep "${CFILTER}"|grep "${AFILTER}"|grep "${VFILTER}"|while read LINE
  do
#  echo "  ${LINE}"
    CLUSTER=`echo ${LINE}|awk -F\; '{print $1}'|grep "nlnaf"`
    VOLUME=`echo ${LINE}|awk -F\; '{print $2}'|grep "_vol"`
    DEST_AGGR=`echo ${LINE}|awk -F\; '{print $3}'|grep "n[0-9][0-9]a[0-9][0-9][0-9][0-9]"`

## echo "  CLUSTER=${CLUSTER} VOLUME=${VOLUME} DEST_AGGR=${DEST_AGGR}"

    if [ "${CLUSTER}" != "" ] && [ "${VOLUME}" != "" ] && [ "${DEST_AGGR}" != "" ]; then

# Get info from cluster(s) if not yet done
    if [ ! -f ${TMP}_vol_${CLUSTER} ]; then
      echo "    Getting (temp) info from ${CLUSTER}"
#   vserver;volume;aggregate;
      ${SSH} ${CLUSTER} "set -units GB -showseparator \";\" ; volume show -field vserver,volume,aggregate" >> ${TMP}_vol_${CLUSTER}
# vserver;volume;state;
      ${SSH} ${CLUSTER} "set -units GB -showseparator \";\" ; volume move show -field vserver,volume,state" >> ${TMP}_vol-move_${CLUSTER}
# aggregate;percent-used;
      ${SSH} ${CLUSTER} "set -units GB -showseparator \";\" ; storage aggregate show -field percent-used" >> ${TMP}_aggr_${CLUSTER}
    fi  # Not ${TMP}_vol_${CLUSTER}

#  VSERVER=`${SSH} ${CLUSTER} "volume show -volume ${VOLUME} -field vserver"| grep ${VOLUME}|awk '{print $1}'`
    VSERVER=`grep ";${VOLUME};" ${TMP}_vol_${CLUSTER}|awk -F\; '{print $1}'`

# Check if volume is already moved to dest-aggr
#  CHECK_DEST_AGGR_VOLUME=`${SSH} ${CLUSTER} "volume show -aggregate ${DEST_AGGR} -volume ${VOLUME} -field volume,aggregate"|grep ${DEST_AGGR}`
    CHECK_DEST_AGGR_VOLUME=`grep ";${VOLUME};" ${TMP}_vol_${CLUSTER}|grep ${DEST_AGGR}|awk -F\; '{print $3}'`

    echo "  CLUSTER=${CLUSTER}| VSERVER=${VSERVER}| VOLUME=${VOLUME}| DEST_AGGR=${DEST_AGGR}| CHECK_DEST_AGGR_VOLUME=${CHECK_DEST_AGGR_VOLUME}| ..."

# When empty (not moved yet. Then start move)
    if [ "${CHECK_DEST_AGGR_VOLUME}" = "" ]; then
#    CHECK_VOLUME_IN_MOVE=`${SSH} ${CLUSTER} "volume move show"|grep ${VOLUME}`
      CHECK_VOLUME_IN_MOVE=`grep ";${VOLUME};" ${TMP}_vol-move_${CLUSTER}|awk -F\; '{print $2}'`
# Check if volume still being moved, then skip for a next one
      echo "  CHECK_VOLUME_IN_MOVE=${CHECK_VOLUME_IN_MOVE}"
      if [ "${CHECK_VOLUME_IN_MOVE}" = "" ]; then
# Check if DEST_AGGR < MAX_AGGR_PERC
#      DEST_AGGR_PERC=`${SSH} ${CLUSTER} "storage aggregate show -aggregate ${DEST_AGGR} -field percent-used"|grep ${DEST_AGGR}|awk '{print $2}'|sed 's/%//g'`
        DEST_AGGR_PERC=`grep "${DEST_AGGR};" ${TMP}_aggr_${CLUSTER}|awk -F\; '{print $2}'|sed 's/%//g'`
        echo "  DEST_AGGR_PERC=${DEST_AGGR_PERC} (${DEST_AGGR})"

# Check NUM_OF_MOVES before start new ones
        NUM_OF_MOVES=`${SSH} ${CLUSTER} "volume move show -field state"| grep healthy|wc -l`
        echo "  NUM_OF_MOVES=${NUM_OF_MOVES}"
        if [ ${NUM_OF_MOVES} -ge ${MAX_MOVES} ]; then
          echo "  Max numbers(${MAX_MOVES}) of moves(${NUM_OF_MOVES}) reached at ${CLUSTER}. Exit(5)"|tee -a ${LOG}
          rm ${TMP}_*
          exit 5
        fi  # NUM_OF_MOVES
        echo "DEST_AGGR_PERC=${DEST_AGGR_PERC} MAX_AGGR_PERC=${MAX_AGGR_PERC}"

        if [ ${DEST_AGGR_PERC} -lt ${MAX_AGGR_PERC} ]; then
          echo "    Starting move at CLUSTER=${CLUSTER} VSERVER=${VSERVER} VOLUME=${VOLUME} DEST_AGGR=${DEST_AGGR} (${DEST_AGGR_PERC}%)"|tee -a ${LOG}
          ${SSH} ${CLUSTER} "volume move start -vserver ${VSERVER} -volume ${VOLUME} -destination-aggregate ${DEST_AGGR}"
        else
# AGGR > MAX
          echo "`date` Destination aggr (${CLUSTER}:${DEST_AGGR}) usage ${DEST_AGGR_PERC}% > max (${MAX_AGGR_PERC}). Move of ${VOLUME} NOT started." | mailx -s ":${HOSTNAME}: WARNING from vol-move-init [${PGM} v${VER}]" ${MAILTO}
          echo "  Mailed WARNING: ${CLUSTER}:${DEST_AGGR} usage ${DEST_AGGR_PERC}% > max (${MAX_AGGR_PERC}). Move of ${VOLUME} NOT started to ${MAILTO}. Exit(6)"|tee -a ${LOG}
          exit 6
        fi
      fi  # CHECK_VOLUME_IN_MOVE
    fi  # CHECK_DEST_AGGR_VOLUME

    else  # CLUSTER <> ""
      echo "  One ore more are empty: CLUSTER=${CLUSTER} VOLUME=${VOLUME} DEST_AGGR=${DEST_AGGR}"
    fi  # CLUSTER <> ""
  done  # cat CSV
  rm ${TMP}_*
fi  # INIT

if [ ${RPT} ]; then
  echo "  Reporting"
  cp /dev/null ${TMP}
  if [ ! -f ${CLUSTERS} ]; then
    echo "  NO CLUSTERS-file found. Exiting ..."|tee -a ${LOG}
    exit 4
  fi
  cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"| while read CLUSTER
  do
    ${SSH} ${CLUSTER} "volume move show -field estimated-completion-time,phase,destination-aggregate,percent-complete" |grep -i replicating|tee -a ${TMP}

  done  # CLUSTER
fi  # RPT

if [ -s ${TMP} ] && [ ${MAIL} ] && [ "${MAILTO}" != "" ]; then
  cp ${TMP} ${ASC}
  date | mailx -a ${ASC} -s ":${HOSTNAME}: Volume move report [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}" |tee -a ${LOG}
fi  # MAIL

echo "`date`: ${PGM} v${VER} finished"|tee -a ${LOG}
exit 0