#!/bin/sh
# File	: cdot_atos_accounting_v3.sh
# By	: Maarten.deBoer@atos.net, 140909
# Subject	: Script to collect accounting information from cDOT filers. 
#		  Per cDOT-cluster (not per vfiler)
#		  In the accounting-v3 way 
#		  Storage classes are determined from the QoS-policy.
#                 Therefore QoS should be set on all volumes.
#                 Lack of QoS-policy will be reported.
# 		  And send it per mail to MSS-accounting-server
#(0.2),211220	: Mod. at "Checks if .ASC-file is NOT empty" (_v3.asc added)
#(0.3),250408	: Add 2nd dest (new consolidations server)
#(0.4),250527	: Add DATE_YMD
#(0.5),250714	: Added SCPHOST3
#(0.6),260324	: Mod. disabled SCPHOST & SCPHOST2, nodes in SSHCMD
#(0.7),260330	: Add ACCT_CSV
#(0.8),260330	: Add update fg_oss with STOG-class (QOS)
#(0.9),260416	: Swam /fsod-test/ & /fsod/. Removed old SCPUSER,SCPHOST,SCPDIR
PGM="`basename $0|cut -d\. -f1`"
VER="0.9"
TMP="/tmp/${TMP}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
DATADIR="${HOME}/data/out"
MAILTO="maarten.deboer@atos.net"
ACCT_DIR="${HOME}/data/accounting"

SCPUSER="mssbatch"
SCPHOST="nlxmssp3"
#SCPHOST=""
SCPDIR="/home/mssbatch/data/recharging/fsod"

CLUSTERS="${HOME}/etc/clusters"
FILTER="[?]*"

# This part (func) is needed for resolving BUSY of "main" cluster interface. 
# Therefore connecting (SSH) to node-interface(s)
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host : Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 2
    /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 2
      /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 2
        /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} sleep 2 ..."
            sleep 2
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}

            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              echo "  EC=${EC} sleep 2 ..."
              sleep 2
              /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}

              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to node ports failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0

            fi  #
          fi  # 
        fi  # 
      fi  # 
    fi  # 
  fi  # 
  rm ${TMPERR}
}

if [ "${1}" != "" ]; then
  FILTER="${1}"
fi

# MAIN
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "  FILTER(1)=${FILTER}"
sleep 1
touch ${TMP}

if [ ! -d ${DATADIR} ]; then
  echo "  NO DATADIR (${DATADIR}). So, exting ... "|tee -a ${LOG}
  exit 4
fi  # DATADIR
if [ ! -d ${ACCT_DIR} ]; then
  echo "  NO ACCT_DIR (${ACCT_DIR}). So, exting ... "|tee -a ${LOG}
  exit 5
fi  # ACCT_DIR

for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep "${FILTER}"`
do
  echo "  ${CLUSTER}"|tee -a ${LOG}
  DATE_YMD=`date +%Y%m%d`
  ACCT_CSV="${ACCT_DIR}/${DATE_YMD}_${CLUSTER}_v3.csv"
  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" > ${ACCT_CSV}
  echo "# vserver;volume;aggregate;total;used;percent-used;type;qos-policy-group;qos-adaptive-policy-group;" >> ${ACCT_CSV}

  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" > ${DATADIR}/${CLUSTER}_v3.asc
# option -field selects the items to be shown. NOT the order. That is FIXED
  SSHCMD ${CLUSTER} 'set -units KB -showseparator ";" ; volume show -state online -field vserver,aggregate,volume,type,total,used,percent-used,qos-policy-group,qos-adaptive-policy-group'|grep -v 'Volume Name'|tee -a ${DATADIR}/${CLUSTER}_v3.asc |tee -a ${ACCT_CSV}

# -----------------------------------------------------------------
# Insert qos-policy-group info ACCT_CSV_2
  ACCT_CSV_2="`echo ${ACCT_CSV}|cut -d\_ -f1-2`_v3-5.csv"
  echo "  ACCT_CSV=${ACCT_CSV} ACCT_CSV_2=${ACCT_CSV_2}"
sleep 2
  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" > ${ACCT_CSV_2}
  echo "# vserver;volume;aggregate;total;used;percent-used;type;qos-policy-group;qos-adaptive-policy-group;" >> ${ACCT_CSV_2}
# Update fg_oss volume with QOS (Storage class) in CSV file
  cat ${ACCT_CSV}|grep -v ^#|while read LINE
  do
# vserver;volume;aggregate;total;used;percent-used;type;qos-policy-group;qos-adaptive-policy-group;
#    echo "${LINE}"
    VSERVER=`echo ${LINE}|cut -d\; -f1`
    VOLUME=`echo ${LINE}|cut -d\; -f2`
    AGGREGATE=`echo ${LINE}|cut -d\; -f3`
    TOTAL=`echo ${LINE}|cut -d\; -f4`
    USED=`echo ${LINE}|cut -d\; -f5`
    PERC=`echo ${LINE}|cut -d\; -f6`
    TYPE=`echo ${LINE}|cut -d\; -f7`
    QOS=`echo ${LINE}|cut -d\; -f8`
    AQOS=`echo ${LINE}|cut -d\; -f9`

# If (a)QOS is unknown ("-") and RW, then try to find QOS
    if [ "${QOS}" = "-" ] && [ "${AQOS}" = "-" ] && [ "${TYPE}" = "RW" ] && [ "`echo ${VSERVER}|grep nlnafs`" != "" ] && [ "`echo ${VOLUME}|grep vol0root`" = "" ]; then
      echo "    CLUSTER=${CLUSTER} VSERVER=${VSERVER} VOLUME=${VOLUME}"
      QOS=`${SSH} ${CLUSTER} "set -showseparator \";\" ; vserver object-store-server bucket show -vserver ${VSERVER} -volume ${VOLUME} -field qos-policy-group"|grep ${VSERVER}|cut -d\; -f3`
      if [ "${QOS}" = "" ]; then
        QOS="-"
      fi
# Show only the replaced info.
      echo "${VSERVER};${VOLUME};${AGGREGATE};${TOTAL};${USED};${PERC};${TYPE};${QOS};${AQOS};" |tee -a ${ACCT_CSV_2}
    else
# Others, just direct (without show/tee) into file
      echo "${VSERVER};${VOLUME};${AGGREGATE};${TOTAL};${USED};${PERC};${TYPE};${QOS};${AQOS};" >> ${ACCT_CSV_2}
    fi  # SSH
  done  # LINE
# -----------------------------------------------------------------


# Checks if .ASC-file is NOT empty
  LINECNT="`cat ${DATADIR}/${CLUSTER}_v3.asc|grep -v ^#|wc -l`"
  if [ ${LINECNT} -lt 5 ];then
    echo "`date` ${DATADIR}/${CLUSTER}_v3.asc has only ${LINECNT} lines. Might be a problem. Do investigate logfile (${HOSTNAME}:${LOG})."|tee -a ${LOG}|mailx -s ":${HOSTNAME}: ERROR in NetApp-accounting for ${CLUSTER} [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed ERROR to ${MAILTO}"|tee -a ${LOG}
  fi

# copy file by using SCP
  if [ "${SCPHOST}" != "" ]; then
    scp ${ACCT_CSV_2} ${SCPUSER}@${SCPHOST}:${SCPDIR}/${CLUSTER}_v3-${DATE_YMD}.asc
    EC=${?}
    echo "`date` Information SCP-ed(${EC}) to ${SCPUSER}@${SCPHOST}:${SCPDIR} "|tee -a ${LOG}
    ${SSH} ${SCPUSER3}@${SCPHOST} ls -l ${SCPDIR}/|tee -a ${LOG} 2>&1
  else
    echo "  NO SCPHOST defined. So, NO copy to remote host."|tee -a ${LOG}
  fi  # ${SCPHOST}
done  # for CLUSTER

# Finish
rm ${TMP}
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0