#!/bin/sh
# File	: cdot_atos_accounting_v4.sh
# By	: Maarten.deBoer@atos.net, 140909
# Subject	: Script to collect accounting information from cDOT filers. 
#		  Per cDOT-cluster (not per vfiler)
#		  In the accounting-v3 way 
#		  Storage classes are determined from the QoS-policy.
#                 Therefore QoS should be set on all volumes.
#                 Lack of QoS-policy will be reported.
# 		  And send it per mail to MSS-accounting-server
# v4		: Added (S3)Object storage volumes (FlexGroups) with QoS
#(0.2),211220	: Mod. at "Checks if .ASC-file is NOT empty" (_v3.asc added)
#(0.3),250408	: Add 2nd dest (new consolidations server)
#(0.4),250527	: Add DATE_YMD
#(0.5),250714	: Added SCPHOST3
#(0.6),250728	: Add. RCFILE & ping
#(0.1),250801	: v4, Add to CSV-file too
#(0.2),250804	: Show on volume-base. 1:volume(flexvol), 2:bucket(flexgroup)
#(0.3),251211	: Mod for accounting between v3 and v4(FlexGroups). ACCNT_V stays 3, added if ${HOST_NRS}
#(0.4),260126	: Mod's
#(0.5),260202	: Mod's
#(0.6),260203	: Disable CLUSTER part:
PGM="`basename $0|cut -d\. -f1`"
VER="0.6"
TMP="/tmp/${TMP}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
DATADIR="${HOME}/data/out"
MAILTO="maarten.deboer@atos.net"
RCFILE="${HOME}/etc/${PGM}.rc"
ACCNT_V="3"
MAIL=""
FILTER="[?]*"
PREFIX="nlnaf|tle|nld"

CLUSTERS="${HOME}/etc/clusters"

#HOST_NRS="1 2 3"
HOST_NRS="3"

SCPUSER[1]="receiver"
SCPHOST[1]="nlxmssp1"
SCPDIR[1]="/home/receiver/import/inventory/cdot/"
#SCPUSER[1]="nl19471"
#SCPHOST[1]="tlenlmgt004"
#SCPDIR[1]="/home/nl19471/data/nas/"

SCPUSER[2]="mssbatch"
SCPHOST[2]="nlxmssscr01"
SCPDIR[2]="/home/mssbatch/data/recharging/fsod"
#SCPUSER[2]="nl19471"
#SCPHOST[2]="tlenlmgt003"
#SCPDIR[2]="/home/nl19471/data/nas/"

SCPUSER[3]="mssbatch"
SCPHOST[3]="nlxmssp3"
SCPDIR[3]="/home/mssbatch/data/recharging/fsod"
#SCPDIR[3]="/home/mssbatch/data/recharging/test"


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 2
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 2
      /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 2
        /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} sleep 2 ..."
            sleep 2
            /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              echo "  EC=${EC} sleep 2 ..."
              sleep 2
              /usr/bin/ssh -n ${1}-13 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Other connection failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0

            fi  # -
          fi  # -
        fi  # -
      fi  # -
    fi  # -
  fi  # -
  rm ${TMPERR}
}


# MAIN
if [ "${1}" != "" ]; then
  FILTER="${1}"
fi  # ${1}
if [ "${2}" != "" ]; then
  PREFIX="${2}"
fi  # ${2}
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
if [ -f ${RCFILE} ]; then
  echo "  RCFILE(${RCFILE}) found. Using ... "|tee -a ${LOG}
  . ${RCFILE}
fi  # RCFILE
echo "  CLUSTERS=${CLUSTERS}"
echo "  DATADIR=${DATADIR}"
echo "  MAILTO=${MAILTO}"
for HOST_NR in ${HOST_NRS}
do
  echo "  SCPUSER[${HOST_NR}]=${SCPUSER[${HOST_NR}]}"
  echo "  SCPHOST[${HOST_NR}]=${SCPHOST[${HOST_NR}]}"
  echo "  SCPDIR[${HOST_NR}]=${SCPDIR[${HOST_NR}]}"
done  # for
echo "  HOST_NRS=${HOST_NRS}"
echo "  FILTER(1)=${FILTER}"
echo "  PREFIX(2)=${PREFIX}"
sleep 1

touch ${TMP}

cat ${CLUSTERS}|grep -v "^#"|grep "${FILTER}"|while read CLUSTER
do
#
# Cluster
  ping -c 1 -W 2 ${CLUSTER}
  EC=${?}
  if [ ${EC} -eq 0 ]; then
    echo "${CLUSTER} ..."
    DATE_YMD=`date +%Y%m%d`
    echo "# `date` (${DATE_YMD}) by ${PGM} v${VER} at ${HOSTNAME}" > ${DATADIR}/${CLUSTER}_v3.asc

    echo "# vserver;volume;aggregate;total;used;percent-used;type;qos-policy-group;qos-adaptive-policy-group;" >> ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv
    echo "vserver;volume;aggregate;total;used;percent-used;type;qos-policy-group;qos-adaptive-policy-group;" >> ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc

#   option -field selects the items to be shown. NOT the order. That is FIXED
#
# Due to separation of volumes into FlexVol and FlexGroup, there are new loops introduced
# 1(1st) for FlexVol's en 1 (2nd) for FlexGroup (Buckets)
# Therefor the volumes are listed on VSERVER-base.

## Then we're missing volume (info) of the clusters
## So, cluster volumes (vol0, MDV are listed by the cluser selection
## CLUSTER part:
#    SSHCMD ${CLUSTER} "set -units KB -showseparator \";\" ; volume show -volume-style-extended flexvol -state online -field vserver,volume,aggregate,type,total,used,percent-used,qos-policy-group,qos-adaptive-policy-group"|grep "${CLUSTER}"|grep 'KB'|tee -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc|tee -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv

# VSERVER-part:
# Without -operational-state running. For SVM-DR volumes
    SSHCMD ${CLUSTER} "set -showseparator \";\" ; vserver show -field vserver"|egrep "${PREFIX}"|awk -F\; '{print $1}'|while read VSERVER
    do
      echo "    VSERVER=${VSERVER}|"
sleep 1

# ----------------------
# FlexVol volumes only
      SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume show -state online -volume-style-extended flexvol -field volume"|grep "${VSERVER}"|awk -F\; '{print $2}'|while read FLEXVOL_VOLUME
      do
        echo "      FLEXVOL_VOLUME=${FLEXVOL_VOLUME}|"
        SSHCMD ${CLUSTER} "set -units KB -showseparator \";\" ; volume show -vserver \"${VSERVER}\" -volume \"${FLEXVOL_VOLUME}\" -state online -field vserver,volume,aggregate,type,total,used,percent-used,qos-policy-group,qos-adaptive-policy-group"|grep 'KB'|tee -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc|tee -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv
      done  # FLEXVOL_VOLUME

# ----------------------
# FlexGroup volumes only
      SSHCMD ${CLUSTER} "set -showseparator \";\" ; volume show -state online -volume-style-extended flexgroup -field volume"|grep "${VSERVER}"|awk -F\; '{print $2}'|while read FLEXGROUP_VOLUME
      do
        echo "      ${FLEXGROUP_VOLUME}"
#fields: vserver(1);bucket(2);type(3);volume(4);size(5);logical-used(6);qos-policy-group(7);
	SSHCMD ${CLUSTER} "set -units KB -showseparator \";\" ; vserver object-store-server bucket show -vserver \"${VSERVER}\" -volume \"${FLEXGROUP_VOLUME}\" -field vserver,volume,qos-policy-group,size,logical-used,type"|awk -F\; '{print $1";"$4";-;"$5";"$6";na%;RW;"$7";-;"}'|grep 'KB'|tee -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc|tee -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv
      done  # FLEXGROUP_VOLUME
#asc: vserver(1);volume(2);aggr(3);total(4);used(5);percent-used(6);type(7);qos-policy-group(8);qos-adaptive-policy-group(9);" 
    done  # VSERVER

    echo "# vserver;volume;(aggr);size;logical-used;(percent-used);(type);qos-policy-group;(qos-adaptive-policy-group);" >> ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv

#   Checks if .ASC-file is NOT empty
    LINECNT="`cat ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc|grep -v ^#|wc -l`"
    if [ "${MAILTO}" != "" ] && [ ${LINECNT} -lt 5 ];then
      echo "`date` ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc has only ${LINECNT} lines. Might be a problem. Do investigate logfile (${HOSTNAME}:${LOG})."|tee -a ${LOG}|mailx -s ":${HOSTNAME}: ERROR in NetApp-accounting for ${CLUSTER} [${PGM} v${VER}]" ${MAILTO}
      echo "  Mailed ERROR to ${MAILTO}"|tee -a ${LOG}
    fi

    if [ "${MAILTO}" != "" ] && [ ${MAIL} ];then
      echo "`date` ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv"| mailx -a ${DATADIR}/${CLUSTER}_v${ACCNT_V}.csv -s ":${HOSTNAME}: NetApp-accounting CSV for ${CLUSTER} [${PGM} v${VER}]" ${MAILTO}
      echo "  Mailed CSV (${CLUSTER}_v${ACCNT_V}.csv) to ${MAILTO}"|tee -a ${LOG}
    fi  # MAILTO && MAIL

#   copy file by using SCP
    if [ "${HOST_NRS}" != "" ]; then
      echo "  Copying accounting data ..."
      for HOST_NR in ${HOST_NRS}
      do
        ping -c 1 -W 2 ${SCPHOST[${HOST_NR}]}
        PEC=${?}
        if [ ${PEC} -eq 0 ]; then
          scp ${DATADIR}/${CLUSTER}_v${ACCNT_V}.asc ${SCPUSER[${HOST_NR}]}@${SCPHOST[${HOST_NR}]}:${SCPDIR[${HOST_NR}]}/${CLUSTER}_v${ACCNT_V}-${DATE_YMD}.asc
  	SEC=${?}
          if [ ${SEC} -eq 0 ]; then
            echo "`date` Information SCP-ed to ${SCPUSER[${HOST_NR}]}@${SCPHOST[${HOST_NR}]}:${SCPDIR[${HOST_NR}]} "|tee -a ${LOG}
            ${SSH} ${SCPUSER[${HOST_NR}]}@${SCPHOST[${HOST_NR}]} ls -l ${SCPDIR[${HOST_NR}]}|tee -a ${LOG} 2>&1
          fi  # SEC=0
        fi  # PEC=0
      done  # HOST
    fi  # HOST_NRS}" != ""

  fi  # ping CLUSTER EC = 0
done  # for CLUSTER

# Finish
rm ${TMP}
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0