#!/bin/sh
# File	: cdot_chk-set_efficiency.sh
# By	: Maarten.deBoer@atos.net, 170920,220817
# Subject	: Script to check (some) efficiency parameter and able to set them
# set -x
#(0.2),220817	: Added USAGE
#(0.3),220818	: Mod's
#(0.4),220818	: Added NFILTER, AFILTER
#(0.5),220822	: Mod's with SET_EFF & add ASC
#(0.6),220822	: Add DO_PROMOTE
#(0.7),220822	: Add MAX_PROGRESS_TIME
#(0.8),220822	: Rename to cdot_chk-set_efficiency.sh (from cdot_chk_efficiency.sh)
#(0.9),220921	: Some mod's
#(0.10),220922	: Added --incident (creation) CRE_INCI
#(0.11),220926	: Add comment & crea-time & PROGRESS_TIME<>""
#(0.12),221003	: Mod mail-message (about Incident)
PGM="`basename $0|cut -d\. -f1`"
VER="0.12"
TMP="/tmp/${TMP}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
MAILTO="maarten.deboer@atos.net"
MAIL=""
CLUSTERS="${HOME}/etc/clusters"
AFILTER="[?]*"
CFILTER="[?]*"
NFILTER="[?]*"
SFILTER="[?]*"
VFILTER="_vol"
SET_EFF=""
ASC="/tmp/${PGM}.asc"
DO_PROMOTE=""
MAX_PROGRESS_TIME=48
CRE_INCI=""

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 2
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 2
      /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 2
        /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} sleep 2 ..."
            sleep 2
            /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}

            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              echo "  EC=${EC} sleep 2 ..."
              sleep 2
              /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}

              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0

            fi  # -010
          fi  # -09
        fi  # -08
      fi  # -06
    fi  # -06
  fi  # -01
  rm ${TMPERR}
}
USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VER}"
  echo "  options           :"
  echo "    -a | --aggr     : Aggregate filter (${AFILTER})"
  echo "    -c | --cluster  : Cluster filter (${CFILTER})"
  echo "    -h | --help     : this help"
  echo "    -i | --incident : Incident creation"
  echo "    -n | --node     : Node filter (${SFILTER})"
  echo "    -m | --mail     : Mail the logging"
  echo "    -s | --svm      : Svm filter (${SFILTER})"
  echo "    -v | --vol      : Volume filter (${VFILTER})"
  echo "    -V              : Version"
  echo "    -x              : set -x"
  echo "    --do_promote    : Do promote a volume when deprioritized"
  echo "    --mailto        : set new MAILTO (${MAILTO})"
  echo "    --max           : mod MAX_progress_time (${MAX_PROGRESS_TIME})"
  echo "    --set_eff       : set efficiency"
}
# Check options
while [ ${#} -gt 0 ]
do
  case ${1} in
    -a | --aggr) AFILTER="${2}"; shift ;;
    -c | --cluster) CFILTER="${2}"; shift ;;
    -n | --node) NFILTER="${2}"; shift ;;
    -s | --svm) SFILTER="${2}"; shift ;;
    -v | --vol) VFILTER="${2}"; shift ;;
    -i | --incident) CRE_INCI=1 ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    --mailto) MAIL=1; MAILTO="${2}"; shift ;;
    --max) MAX_PROGRESS_TIME="${2}"; shift ;;
    --set_eff) SET_EFF=1 ;;
    --do_promote) DO_PROMOTE=1 ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done  # while


# MAIN
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG} ${TMP}.m
echo "CLUSTERS=${CLUSTERS}"| tee -a ${TMP}.m
echo "MAILTO=${MAILTO}"| tee -a ${TMP}.m
echo "MAIL=${MAIL}"| tee -a ${TMP}.m
echo "MAX_PROGRESS_TIME=${MAX_PROGRESS_TIME}"| tee -a ${TMP}.m
echo "AFILTER=${AFILTER}"| tee -a ${TMP}.m
echo "CFILTER=${CFILTER}"| tee -a ${TMP}.m
echo "NFILTER=${NFILTER}"| tee -a ${TMP}.m
echo "SFILTER=${SFILTER}"| tee -a ${TMP}.m
echo "VFILTER=${VFILTER}"| tee -a ${TMP}.m
echo "SET_EFF=${SET_EFF}"| tee -a ${TMP}.m
echo "DO_PROMOTE=${DO_PROMOTE}"| tee -a ${TMP}.m
echo "CRE_INCI=${CRE_INCI}"| tee -a ${TMP}.m
sleep 1

echo "  Filters: A=${AFILTER} C=${CFILTER} N=${NFILTER} S=${SFILTER} V=${VFILTER} MAIL=${MAIL} MAX_PROGRESS_TIME=${MAX_PROGRESS_TIME} SET_EFF=${SET_EFF} DO_PROMOTE=${DO_PROMOTE} CRE_INCI=${CRE_INCI}" | tee -a ${LOG}

touch ${TMP}
for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"`
do
  echo "  ${CLUSTER} aggr-list & volume-list ..."
  SSHCMD ${CLUSTER} "set -showseparator \";\" ; aggr show -field aggregate,storage-type,node,root" |grep ${CLUSTER} > ${TMP}_aggr-show
  SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume show -field node,vserver,volume,aggregate,type" |grep ${CLUSTER} > ${TMP}_vol-show

  SSHCMD ${CLUSTER} 'vserver show -operational-state running -field vserver'|grep nl|grep "${SFILTER}"|while read VSERVER REST
  do
    echo "  ${CLUSTER}/${VSERVER} ..."

    SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume efficiency show -vserver ${VSERVER} -field policy,compression,inline-compression,inline-dedupe,state,progress"|grep "${VFILTER}"|while read LINE
    do
# vserver;volume;state;progress;policy;compression;inline-compression;inline-dedupe
      VSERVER=`echo ${LINE}|awk -F\; '{print $1}'|grep nlnaf`
      VOLUME=`echo ${LINE}|awk -F\; '{print $2}'|grep vol`
      STATE=`echo ${LINE}|awk -F\; '{print $3}'`
      PROGRESS_IDLE=`echo ${LINE}|awk -F\; '{print $4}'|grep -i idle`
      POLICY=`echo ${LINE}|awk -F\; '{print $5}'`
      COMPRESSION=`echo ${LINE}|awk -F\; '{print $6}'`
      INLINE_COMPRESSION=`echo ${LINE}|awk -F\; '{print $7}'`
      INLINE_DEDUPE=`echo ${LINE}|awk -F\; '{print $8}'`

      VOL_AGGR=`grep ${VOLUME} ${TMP}_vol-show|grep ${AFILTER}|awk -F\; '{print $3}'`
      AGGR_TYPE=`grep ${VOL_AGGR} ${TMP}_aggr-show|awk -F\; '{print $2}'`
      VOL_NODE=`grep ${VOLUME} ${TMP}_vol-show|grep "${NFILTER}"|awk -F\; '{print $5}'`
      VOL_TYPE=`grep ${VOLUME} ${TMP}_vol-show|awk -F\; '{print $4}'`

      if [ "${VOL_AGGR}" != "" ] && [ "${VOL_NODE}" != "" ]; then
# When node and aggr is found by selection
#echo "    ${VSERVER};${VOLUME};${VOL_TYPE};${VOL_NODE};${VOL_AGGR};${STATE};${PROGRESS};${POLICY};${COMPRESSION};${INLINE_COMPRESSION};${INLINE_DEDUPE};"


# Progess ("Idle for 247:06:14")
        PROGRESS_TIME=`echo ${PROGRESS_IDLE}|awk '{print $3}'|cut -d\: -f1`
        if [ "${PROGRESS_TIME}" != "" ] && [ ${PROGRESS_TIME} -gt ${MAX_PROGRESS_TIME} ]; then
          VOL_COMM_CRE_TIME=`SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume show -volume ${VOLUME} -field comment,create-time"|grep ${VOLUME}|awk -F\; '{print $3,$4}'`
          echo "    Volume ${CLUSTER}:${VSERVER}:${VOLUME} efficiency progress > ${MAX_PROGRESS_TIME}hr (${PROGRESS_TIME}hr), ${VOL_COMM_CRE_TIME}."|tee -a ${LOG} ${TMP}
        fi  # PROGRESS_TIME} -gt 48 

# Enables deduplication on the volume specified.
        if [ "${STATE}" = "Disabled" ]; then
          echo "    Vol efficiency=Disabled on ${CLUSTER}:${VSERVER}:${VOLUME}|"|tee -a ${LOG} ${TMP}
          if [ ${SET_EFF} ]; then
            echo "    Enabling (efficiency on) deduplication"|tee -a ${LOG}
            SSHCMD ${CLUSTER} "volume efficiency on -vserver ${VSERVER} -volume ${VOLUME}" 1> ${TMP}_out
            grep -i "error" ${TMP}_out
            EC=${?}
            if [ ${EC} -eq 0 ]; then
              echo "    ERROR : volume efficiency on -vserver ${VSERVER} -volume ${VOLUME}"| tee -a ${LOG} ${TMP}
              cat ${TMP}_out |tee -a ${LOG}
            fi  # EC > 0
          fi  # SET_EFF
        fi  # STATE}" = "Disabled

# policy=dedup_daily
        if [ "${POLICY}" != "dedup_daily" ]; then
          echo "    POLICY<>dedup_daily (${POLICY}) on ${CLUSTER}:${VSERVER}:${VOLUME}"|tee -a ${LOG} ${TMP}
          CHK_POLICY=`SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume efficiency policy show -vserver ${VSERVER} -policy dedup_daily -field policy,schedule,comment"|grep dedup_daily`
          if [ ${SET_EFF} ]; then
            if [ "${CHK_POLICY}" = "" ]; then
# Check if dedup_daily policy is at Vserver. Create one
              echo "    No dedup_daily policy on ${CLUSTER}:${VSERVER}"|tee -a ${LOG} ${TMP}
              echo "    Creating Eff-policy (dedup_daily) onto ${VSERVER}"|tee -a ${LOG}
              SSHCMD ${CLUSTER} "volume efficiency policy create -vserver ${VSERVER} -policy dedup_daily -schedule hour20 -comment \"Created by ${PGM} v${VER} on `date`\" "
# schedule = dedup_daily
            fi  # CHK_POLICY
            echo "    Setting POLICY=dedup_daily to ${CLUSTER}:${VSERVER}:${VOLUME}"|tee -a ${LOG}
            SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -policy dedup_daily" 1> ${TMP}_out
            grep -i "error" ${TMP}_out
            EC=${?}
            if [ ${EC} -eq 0 ]; then
              echo "    ERROR (${CLUSTER}) : volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -policy dedup_daily"| tee -a ${LOG} ${TMP}
              cat ${TMP}_out |tee -a ${LOG} ${TMP}
              if [ ${DO_PROMOTE} ]; then
                echo "      volume promote"|tee -a ${LOG} ${TMP}
                SSHCMD ${CLUSTER} "set -privilege advanced;volume efficiency promote -vserver ${VSERVER} -volume ${VOLUME}" 1> ${TMP}_out
                SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -policy dedup_daily" 1> ${TMP}_out
                cat ${TMP}_out |tee -a ${LOG} ${TMP}
              fi  # DO_PROMOTE
            fi  # EC > 0
          fi  # SET_EFF
        fi  # POLICY} != "dedup_daily" ] && [ ${SET_EFF}

# -compression
        if [ "${COMPRESSION}" = "false" ] && [ ${SET_EFF} ]; then
          echo "    COMPRESSION=false on ${CLUSTER}:${VSERVER}:${VOLUME}"| tee -a ${LOG} ${TMP}
          if [ ${SET_EFF} ]; then
            SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -compression true" 1> ${TMP}_out
            grep -i "error" ${TMP}_out
            EC=${?}
            if [ ${EC} -eq 0 ]; then
              echo "    ERROR : volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -compression true"| tee -a ${LOG} ${TMP}
              cat ${TMP}_out |tee -a ${LOG} ${TMP}
              if [ ${DO_PROMOTE} ]; then
                echo "      volume promote"|tee -a ${LOG} ${TMP}
                SSHCMD ${CLUSTER} "set -privilege advanced;volume efficiency promote -vserver ${VSERVER} -volume ${VOLUME}" 1> ${TMP}_out
                SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -compression true" 1> ${TMP}_out
                cat ${TMP}_out |tee -a ${LOG} ${TMP}
              fi  # DO_PROMOTE
            fi  # EC > 0
          fi  # SET_EFF
        fi  # COMPRESSION} = "false"

# -inline-compression
        if [ "${INLINE_COMPRESSION}" = "false" ] && [ ${SET_EFF} ]; then
          echo "    INLINE_COMPRESSION=false on ${VSERVER}:${VOLUME}(${VOL_TYPE}) on ${AGGR_NAME}(${AGGR_TYPE})"|tee -a ${LOG} ${TMP}
          
          if [ ${SET_EFF} ]; then
            echo "    Setting INLINE_COMPRESSION"|tee -a ${LOG}
            SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -inline-compression true" 1> ${TMP}_out
            grep -i "error" ${TMP}_out
            EC=${?}
            if [ ${EC} -eq 0 ]; then
              echo "    ERROR : volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -inline-compression true"| tee -a ${LOG} ${TMP}
              cat ${TMP}_out |tee -a ${LOG} ${TMP}
              if [ ${DO_PROMOTE} ]; then
                echo "      volume promote"|tee -a ${LOG} ${TMP}
                SSHCMD ${CLUSTER} "set -privilege advanced;volume efficiency promote -vserver ${VSERVER} -volume ${VOLUME}" 1> ${TMP}_out
                SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -inline-compression true" 1> ${TMP}_out
                cat ${TMP}_out |tee -a ${LOG} ${TMP}
              fi  # DO_PROMOTE
            fi  # EC > 0
          fi  # SET_EFF
        fi  # INLINE_COMPRESSION} = "false"

# inline-dedupe
        if [ "${INLINE_DEDUPE}" = "false" ]; then
#echo "    VOL_TYPE=${VOL_TYPE} VOL_AGGR=${VOL_AGGR} AGGR_TYPE=${AGGR_TYPE}"
          if [ "${VOL_TYPE}" = "RW" ] && [ "${AGGR_TYPE}" = "ssd" ]; then
            echo "    INLINE_DEDUPE=false on ${VSERVER}:${VOLUME}(${VOL_TYPE}) on ${AGGR_NAME}(${AGGR_TYPE})"|tee -a ${LOG} ${TMP}
            if [ ${SET_EFF} ]; then
              echo "    Setting INLINE_DEDUPE=true "|tee -a ${LOG}
              SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -inline-dedupe true" 1> ${TMP}_out
              grep -i "error" ${TMP}_out
              EC=${?}
              if [ ${EC} -eq 0 ]; then
                echo "    ERROR : volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -inline-compression true"| tee -a ${LOG} ${TMP}
                cat ${TMP}_out |tee -a ${LOG} ${TMP}
                if [ ${DO_PROMOTE} ]; then
                  echo "      volume promote"|tee -a ${LOG} ${TMP}
                  SSHCMD ${CLUSTER} "set -privilege advanced;volume efficiency promote -vserver ${VSERVER} -volume ${VOLUME}" 1> ${TMP}_out
                  SSHCMD ${CLUSTER} "volume efficiency modify -vserver ${VSERVER} -volume ${VOLUME} -inline-dedupe true" 1> ${TMP}_out
                  cat ${TMP}_out |tee -a ${LOG} ${TMP}
                fi  # DO_PROMOTE
              fi  # EC > 0
            fi  # SET_EFF
          fi  # VOL_TYPE}" = "RW" & ssd

        fi # INLINE_DEDUPE

      fi  # VOL_AGGR}" != "" ] && [ "${VOL_NODE}"

    done  # LINE

  done  # VSERVER
  rm ${TMP}_*
done  # for CLUSTER

if [ ${CRE_INCI} ]; then
  echo "  Creating incident"|tee -a ${LOG}
# For IMI2ESB
  IMI_QUEUE="${HOME}/imi2esb/queue"
  IMI_UPLOAD="${HOME}/imi2esb/bin/imi_upload.sh"
  ESBSTAT_URL="https://btn.sapart.atos-srv.net/ART/esbstat.pl?action=info"
# For SNOW
  ESB_USER="sdp"
  REQ_TYPE="Incident"
  SEVNR=3
  REQ_AREA="Storage;Storage;Other"
  GROUP="NL.Storage.FSOD"
  ORG="Atos Netherlands"
  CI="${HOSTNAME}-VSR"

  SUBJECT="Storage Efficiency check [${PGM} v${VER}]"
  MESSAGE="`date` ${PGM}: Filters: A=${AFILTER} C=${CFILTER} N=${NFILTER} S=${SFILTER} V=${VFILTER} MAIL=${MAIL} MAX_PROGRESS_TIME=${MAX_PROGRESS_TIME} SET_EFF=${SET_EFF} DO_PROMOTE=${DO_PROMOTE} CRE_INCI=${CRE_INCI}"
  EXT_REF="${ESB_USER}_`date +%Y%m%d%H%M%S`"
  TICKET="${IMI_QUEUE}/${EXT_REF}.imi"
  ATTMENT="${IMI_QUEUE}/${EXT_REF}.att.file.txt"

# Create ticket
  echo "@PROXYUSER:${ESB_USER}" > ${TICKET}
  echo "@EXTERNALREFERENCE:${EXT_REF}" >> ${TICKET}
  echo "@REQUESTTYPE:${REQ_TYPE}" >> ${TICKET}
  echo "@SEVERITY:${SEVNR}" >> ${TICKET}
  echo "@REQUESTAREA:${REQ_AREA}" >> ${TICKET}
  echo "@GROUP:${GROUP}" >> ${TICKET}
  echo "@ORGANISATION:${ORG}" >> ${TICKET}
  echo "@CI:${CI}" >> ${TICKET}
  echo "${SUBJECT} (${EXT_REF})." >> ${TICKET}
  echo "" >> ${TICKET}
  echo "${MESSAGE}" >> ${TICKET}

  if [ -s ${TMP} ]; then
    cat ${TMP} >> ${ATTMENT}
    echo "See attached file for more details." >> ${TICKET}
  fi  # -s TMP

  echo "" >> ${TICKET}
  echo "EXTERNALREFERENCE=${EXT_REF}" >> ${TICKET}
  echo "" >> ${TICKET}
  echo "@ENDOFUSDMESSAGE:" >> ${TICKET}

# Send info to ESB
  if [ -x ${IMI_UPLOAD} ]; then
    echo "  Sending info to ESB ..."|tee -a ${LOG}
    ${IMI_UPLOAD}
  fi  # IMI_UPLOAD

  echo "  Check status of tickets at https://btn.sapart.atos-srv.net/ART/esbstat.pl?&serie=sdp" 

#  if [ ${MAIL} ]; then
#    echo "  Incident created. Check status of tickets at https://btn.sapart.atos-srv.net/ART/esbstat.pl?&serie=sdp" >> mailx -s ":${HOSTNAME}: Storage Efficiency check [${PGM} v${VER}]" ${MAILTO}
#  fi  # MAIL

fi  # CRE_INCI


if [ ${MAIL} ] && [ -s ${TMP} ]; then
  cp ${TMP} ${ASC}
  ASC_LINE=`cat ${ASC}|wc -l`
  echo "${ASC_LINE} lines in attached file." >> ${TMP}.m
  echo "" >> ${TMP}.m
  if [ ${CRE_INCI} ]; then
    echo "" >> ${TMP}.m
    echo "! Incident will be created too. Look for it into SNOW !" >> ${TMP}.m
    echo "" >> ${TMP}.m
    echo "  Check status of tickets at https://btn.sapart.atos-srv.net/ART/esbstat.pl?&serie=sdp" >> ${TMP}.m
    echo "" >> ${TMP}.m
  fi  # CRE_INCI
  cat ${TMP}.m | mailx -a ${ASC} -s ":${HOSTNAME}: Storage Efficiency check [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
fi  # MAIL


# Finish
rm ${TMP} ${TMP}.m
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0