#!/bin/sh
# File	: cdot_rpt_efficiency.sh
# By	: Maarten.deBoer@atos.net, 170919
# Subject	: Script to collect storage efficiency at NetApp
#(0.2)	: Mod; /SVM in stead of /volume
#(0.3),171109	: Added ${1} -> CFILTER
#(0.4),210923	: Added USAGE
#(0.5),210929	: Mod's
#(0.6),210929	: Add EXCL_FILE
#(0.7),211020	: Add aggr show, NFILTER
PGM="`basename $0|cut -d\. -f1`"
VER="0.7"
TMP="/tmp/${TMP}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
DATADIR="${HOME}/data/out"
MAILTO="maarten.deboer@atos.net"
MAIL=""
CLUSTERS="${HOME}/etc/clusters"
CSV="${PGM}-vol_`date +%Y-%m-%d-%H-%M-%S`.csv"
AGGR_CSV="${PGM}-aggr_`date +%Y-%m-%d-%H-%M-%S`.csv"
AFILTER="[?]*"
CFILTER="[?]*"
SFILTER="[?]*"
NFILTER="[?]*"
#VFILTER="[?]*"
VFILTER="nl"
EXCL_FILE="${HOME}/etc/${PGM}.excl"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 2
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 2
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 2
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} sleep 2 ..."
            sleep 2
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}

            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              echo "  EC=${EC} sleep 2 ..."
              sleep 2
              /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}

              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0

            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "  Version: ${VER}"
  echo "  options          :"
  echo "    -a             : Aggregate filter (${AFILTER})"
  echo "    -c             : Cluster filter (${CFILTER})"
  echo "    -n             : Node filter (${NFILTER})"
  echo "    -s             : Svm filter (${SFILTER})"
  echo "    -v             : Volume filter (${VFILTER})"
  echo "    -m | --mail    : Mail the logging"
  echo "    -h | --help    : this help"
  echo "    -V             : Version"
  echo "    -x             : set -x"
  echo "    --mailto       : set new MAILTO (${MAILTO})"
}
# Check options
while [ ${#} -gt 0 ]
  do
  case ${1} in
    -a) AFILTER=${2}; shift ;;
    -c) CFILTER=${2}; shift ;;
    -s) SFILTER=${2}; shift ;;
    -v) VFILTER=${2}; shift ;;
    -n) NFILTER=${2}; shift ;;
    -m | --mail) MAIL=1 ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    --mailto) MAIL=1; MAILTO=${2}; shift ;;
    *)  echo "Option ${1} not known."; USAGE; exit 1 ;;
  esac
    shift
done


# MAIN
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "CLUSTERS=${CLUSTERS}"
echo "CSV=${CSV}"
echo "AFILTER=${AFILTER}"
echo "CFILTER=${CFILTER}"
echo "NFILTER=${NFILTER}"
echo "SFILTER=${SFILTER}"
echo "VFILTER=${VFILTER}"
echo "MAILTO=${MAILTO}"
sleep 1
touch ${TMP}

if [ ! -f ${EXCL_FILE} ]; then
  touch ${EXCL_FILE}
  echo "  No EXCL_FILE (${EXCL_FILE}) found. So, created (empty)"|tee -a ${LOG}
fi

echo "CLUSTER;VOL AGGREGATE;VOL VSERVER;VOL VOL NAME;VOL TOTAL (GB);VOL USED (GB);VOL LOGICAL_USED (GB);VOL SPACE_GUARANTEE (GB);VOL SIS SPACE SAVED (GB);VOL SIS SPACE SAVED PCT (%);VOL DEDUPE SPACE SAVED (GB);VOL DEDUPE SPACE SAVED PCT (%);VOL COMPRESSION SPACE SAVED (GB);VOL COMPRESSION SPACE SAVED PCT (%);;EFF STATE;EFF POLICY;EFF COMPRESSION;EFF INLINE COMPRESSION;EFF INLINE DEDUPE;EFF DATA COMPACTION;EFF CROSS VOLUME INLINE DEDUPE;EFF CROSS VOLUME BACKGROUND DEDUPE;EFF EXTENDED COMPRESSED DATA;EFF DATA COMPACTION;ENABLED EFFICIENCY FEATURES STR;" > ${TMP}

for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"`
do
#  echo "${CLUSTER} ..."
  SSHCMD ${CLUSTER} 'vserver show -operational-state running -field vserver'|grep nl|grep "${SFILTER}"|while read VSERVER REST
  do
    echo "${CLUSTER}/${VSERVER} ..."
# Volume
    SSHCMD ${CLUSTER} "set -showseparator \";\" ;vol show -vserver ${VSERVER} -field volume,aggregate"|grep ${VFILTER}|grep ${AFILTER}|grep -v -f ${EXCL_FILE}|awk -F\; '{print $2}'|sort|while read VOLUME REST
    do
      echo "${CLUSTER}/${VSERVER}:${VOLUME} ..."

#vserver;volume;aggregate;total;used;space-guarantee;sis-space-saved;sis-space-saved-percent;dedupe-space-saved;dedupe-space-saved-percent;compression-space-saved;compression-space-saved-percent;logical-used;

      LINE=`SSHCMD ${CLUSTER} "set -units GB -showseparator \";\";volume show -vserver ${VSERVER} -volume ${VOLUME} -field vserver,volume,total,compression-space-saved,logical-used,used,aggregate,sis-space-saved,sis-space-saved-percent,dedupe-space-saved,space-guarantee,dedupe-space-saved-percent,compression-space-saved-percent"|grep ${VSERVER}|head -1`
      if [ "${LINE}" != "" ]; then
        VOL_VSERVER=`echo ${LINE}|awk -F\; '{print $1}'`
        VOL_VOL_NAME=`echo ${LINE}|awk -F\; '{print $2}'`
        VOL_AGGREGATE=`echo ${LINE}|awk -F\; '{print $3}'`
        VOL_TOTAL=`echo ${LINE}|awk -F\; '{print $4}'|sed 's/GB//g'`
        VOL_USED=`echo ${LINE}|awk -F\; '{print $5}'|sed 's/GB//g'`
        VOL_SPACE_GUARANTEE=`echo ${LINE}|awk -F\; '{print $6}'`
        VOL_SIS_SPACE_SAVED=`echo ${LINE}|awk -F\; '{print $7}'|sed 's/GB//g'`
        VOL_SIS_SPACE_SAVED_PCT=`echo ${LINE}|awk -F\; '{print $8}'|sed 's/\%//g'`
        VOL_DEDUPE_SPACE_SAVED=`echo ${LINE}|awk -F\; '{print $9}'|sed 's/GB//g'`
        VOL_DEDUPE_SPACE_SAVED_PCT=`echo ${LINE}|awk -F\; '{print $10}'|sed 's/\%//g'`
        VOL_COMPRESSION_SPACE_SAVED=`echo ${LINE}|awk -F\; '{print $11}'|sed 's/GB//g'`
        VOL_COMPRESSION_SPACE_SAVED_PCT=`echo ${LINE}|awk -F\; '{print $12}'|sed 's/\%//g'`
        VOL_LOGICAL_USED=`echo ${LINE}|awk -F\; '{print $13}'|sed 's/GB//g'`
      else
        VOL_VSERVER=""
        VOL_VOL_NAME=""
        VOL_AGGREGATE=""
        VOL_TOTAL=""
        VOL_USED=""
        VOL_SPACE_GUARANTEE=""
        VOL_SIS_SPACE_SAVED=""
        VOL_SIS_SPACE_SAVED_PCT=""
        VOL_DEDUPE_SPACE_SAVED=""
        VOL_DEDUPE_SPACE_SAVED_PCT=""
        VOL_COMPRESSION_SPACE_SAVED=""
        VOL_LOGICAL_USED=""
      fi

# vserver;volume;state;progress;policy;compression;inline-compression;inline-dedupe;data-compaction;cross-volume-inline-dedupe;cross-volume-background-dedupe;extended-compressed-data;
      LINE=`SSHCMD ${CLUSTER} "set -showseparator \";\";vol efficiency show -vserver ${VSERVER} -volume ${VOLUME} -field state,progress,policy,compression,inline-compression,inline-dedupe,compression,data-compaction,cross-volume-inline-dedupe,cross-volume-background-dedupe,extended-compressed-data"|grep ${VSERVER}|head -1`

      if [ "${LINE}" != "" ]; then
        EFF_STATE=`echo ${LINE}|awk -F\; '{print $3}'`
        EFF_PROGRESS=`echo ${LINE}|awk -F\; '{print $4}'`
        EFF_POLICY=`echo ${LINE}|awk -F\; '{print $5}'`
#
        EFF_COMPRESSION=`echo ${LINE}|awk -F\; '{print $6}'`
        EFF_INLINE_COMPRESSION=`echo ${LINE}|awk -F\; '{print $7}'`
        EFF_INLINE_DEDUPE=`echo ${LINE}|awk -F\; '{print $8}'`
        EFF_DATA_COMPACTION=`echo ${LINE}|awk -F\; '{print $9}'`
        EFF_CROSS_VOLUME_INLINE_DEDUPE=`echo ${LINE}|awk -F\; '{print $10}'`
        EFF_CROSS_VOLUME_BACKGROUND_DEDUPE=`echo ${LINE}|awk -F\; '{print $11}'`
        EFF_EXTENDED_COMPRESSED_DATA=`echo ${LINE}|awk -F\; '{print $12}'`
        EFF_DATA_COMPACTION=`echo ${LINE}|awk -F\; '{print $13}'`
      else
        EFF_STATE=""
        EFF_PROGRESS=""
        EFF_POLICY=""
        EFF_COMPRESSION=""
        EFF_INLINE_COMPRESSION=""
        EFF_INLINE_DEDUPE=""
        EFF_DATA_COMPACTION=""
        EFF_CROSS_VOLUME_INLINE_DEDUPE=""
        EFF_CROSS_VOLUME_BACKGROUND_DEDUPE=""
        EFF_EXTENDED_COMPRESSED_DATA=""
        EFF_DATA_COMPACTION=""
      fi

      ENABLED_EFFICIENCY_FEATURES_STR=""

      if [ "${EFF_INLINE_DEDUPE}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Inline Dedupe"
      fi
      if [ "${EFF_CROSS_VOLUME_INLINE_DEDUPE}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Cross Volume Inline Deduplication"
      fi
      if [ ${VOL_DEDUPE_SPACE_SAVED_PCT} -gt 0 ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Deduplication"
      fi
      if [ "${EFF_COMPRESSION}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Compression"
      fi
      if [ "${EFF_DATA_COMPACTION}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Data Compaction"
      fi
      if [ "${EFF_INLINE_COMPRESSION}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Inline Compression"
      fi
      if [ "${EFF_CROSS_VOLUME_BACKGROUND_DEDUPE}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Cross Volume Background Dedupe"
      fi
      if [ "${EFF_EXTENDED_COMPRESSED_DATA}" = "true" ]; then
        ENABLED_EFFICIENCY_FEATURES_STR="${ENABLED_EFFICIENCY_FEATURES_STR}, Extended Compressed Data"
      fi

      ENABLED_EFFICIENCY_FEATURES_STR="."

#      echo "${CLUSTER};${VOL_AGGREGATE};${VOL_VSERVER};${VOL_VOL_NAME};${VOL_TOTAL};${VOL_USED};${VOL_LOGICAL_USED};${VOL_SPACE_GUARANTEE};${VOL_SIS_SPACE_SAVED};${VOL_SIS_SPACE_SAVED_PCT};${VOL_DEDUPE_SPACE_SAVED};${VOL_DEDUPE_SPACE_SAVED_PCT};${VOL_COMPRESSION_SPACE_SAVED};${VOL_COMPRESSION_SPACE_SAVED_PCT};;${EFF_STATE};${EFF_POLICY};  EFF_COMPRESSION ;  EFF_INLINE_COMPRESSION ;  EFF_INLINE_DEDUPE ;  EFF_DATA_COMPACTION ;  EFF_CROSS_VOLUME_INLINE_DEDUPE ;  EFF_CROSS_VOLUME_BACKGROUND_DEDUPE ;  EFF_EXTENDED_COMPRESSED_DATA ;  EFF_DATA_COMPACTION ;  ENABLED_EFFICIENCY_FEATURES_STR ;" |tee -a ${TMP}
      echo "${CLUSTER};${VOL_AGGREGATE};${VOL_VSERVER};${VOL_VOL_NAME};${VOL_TOTAL};${VOL_USED};${VOL_LOGICAL_USED};${VOL_SPACE_GUARANTEE};${VOL_SIS_SPACE_SAVED};${VOL_SIS_SPACE_SAVED_PCT};${VOL_DEDUPE_SPACE_SAVED};${VOL_DEDUPE_SPACE_SAVED_PCT};${VOL_COMPRESSION_SPACE_SAVED};${VOL_COMPRESSION_SPACE_SAVED_PCT};;${EFF_STATE};${EFF_POLICY};  ;  ;  ;  ;  ;  ;  ;  ;${ENABLED_EFFICIENCY_FEATURES_STR};" |tee -a ${TMP}

    done  # VOLUME

    echo "aggregate;node;total-storage-efficiency-ratio;total-data-reduction-efficiency-ratio;volume-logical-used;volume-physical-used;volume-efficiency-saved-ratio;volume-compression-saved-ratio;volume-vbn-zero-saved;volume-data-reduction-storage-efficiency-ratio;snapshot-volume-data-reduction-storage-efficiency-ratio;flexclone-volume-data-reduction-storage-efficiency-ratio;snapshot-flexclone-volume-data-reduction-storage-efficiency-ratio; " > ${TMP}.aggr-eff

    SSHCMD ${CLUSTER} "set -showseparator \";\" ;aggr show -node ${NODE} -field aggregate,node"|grep ${AFILTER}|grep -v -f ${EXCL_FILE}|awk -F\; '{print $2}'|sort|while read AGGR REST
    do
      echo "  ${CLUSTER}${AGGR} ..."
      SSHCMD ${CLUSTER} "set -showseparator \";\" ;aggr show-efficiency -aggregate ${AGGR} -node ${NFILTER} -field aggregate,node,total-storage-efficiency-ratio,total-data-reduction-efficiency-ratio,volume-logical-used,volume-physical-used,volume-efficiency-saved-ratio,volume-compression-saved-ratio,volume-vbn-zero-saved,volume-data-reduction-storage-efficiency-ratio,snapshot-volume-data-reduction-storage-efficiency-ratio,flexclone-volume-data-reduction-storage-efficiency-ratio,snapshot-flexclone-volume-data-reduction-storage-efficiency-ratio" >> ${TMP}.aggr-eff


    done  # AGGR

  done  # VSERVER
done  # for CLUSTER

if [ ${MAIL} ]; then
  cp ${TMP} /tmp/${CSV}
  cp ${TMP}.aggr-eff /tmp/${AGGR_CSV}
  echo "CLUSTERS=${CLUSTERS}" >> ${TMP}.mail
  echo "CSV=${CSV}" >> ${TMP}.mail
  echo "AFILTER=${AFILTER}" >> ${TMP}.mail
  echo "CFILTER=${CFILTER}" >> ${TMP}.mail
  echo "NFILTER=${NFILTER}" >> ${TMP}.mail
  echo "SFILTER=${SFILTER}" >> ${TMP}.mail
  echo "VFILTER=${VFILTER}" >> ${TMP}.mail
  echo "MAILTO=${MAILTO}" >> ${TMP}.mail
  echo "`date`" >> ${TMP}.mail
  cat ${TMP}.mail | mailx -a /tmp/${CSV} -a /tmp/${AGGR_CSV} -s ":${HOSTNAME}: Efficiency report [${PGM} v${VER}]" ${MAILTO}
  echo "  ${CSV} mailed to ${MAILTO}"|tee -a ${LOG}
  cat ${TMP}.mail
fi  # MAIL

# Finish
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
rm ${TMP} /tmp/${CSV}
exit 0