#!/bin/sh
# File	: cdot_atos_accounting_v2.sh
# By	: Maarten.deBoer@atos.net, 140909
# Subject	: Script to collect accounting information from cDOT filers. 
#		  Per cDOT-cluster (not per vfiler)
#		  In the accounting-v2 way 
#		  NO Storage-class in volumename. But combined Aggrname + Volumename  
# 		  And send it per mail to MSS-accounting-server
#Mod 0.2	: scp in stead of mailx
#(0.3)	: Added ls -l at SCPREMOTE
#(0.4),170912	: Added SSHCMD. Because sometimes ssh to cluster-address fails
#		  Added check for less (then 5) lines in .ASC-files and MAILTO
#(0.5),250408	: Add 2nd dest (new consolidations server)
#(0.6),250527	: Add DATE_YMD
#
PGM="`basename $0|cut -d\. -f1`"
VER="0.6"
TMP="/tmp/${TMP}.$$"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
HOSTNAME="`hostname|cut -d\. -f1`"
DATADIR="${HOME}/data/out"
MAILTO="geralt.somsen@atos.net maarten.deboer@atos.net"

SCPUSER="receiver"
SCPHOST="nlxmssp1"
SCPDIR="/home/receiver/import/inventory/cdot/"

SCPUSER2="mssbatch"
SCPHOST2="nlxmssscr01"
SCPDIR2="/home/mssbatch/data/recharging/fsod"

CLUSTERS="${HOME}/etc/clusters"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 2
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 2
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 2
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} sleep 2 ..."
            sleep 2
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}

            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              echo "  EC=${EC} sleep 2 ..."
              sleep 2
              /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}

              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0

            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}


# MAIN
touch ${TMP}
echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}

for CLUSTER in `cat ${CLUSTERS}|grep -v ^#`
do
  echo "${CLUSTER} ..."
  DATE_YMD=`date +%Y%m%d`
  echo "# `date` by ${PGM} v${VER} at ${HOSTNAME}" > ${DATADIR}/${CLUSTER}.asc
# option -field selects the items to be shown. NOT the order. That is FIXED
  SSHCMD ${CLUSTER} 'set -units KB -showseparator ";" ; volume show -state online -field vserver,aggregate,volume,type,total,used,percent-used'|grep -v 'Volume Name'|tee -a ${DATADIR}/${CLUSTER}.asc

# Checks if .ASC-file is NOT empty
  LINECNT="`cat ${DATADIR}/${CLUSTER}.asc|grep -v ^#|wc -l`"
  if [ ${LINECNT} -lt 5 ];then
    echo "`date` ${DATADIR}/${CLUSTER}.asc has only ${LINECNT} lines. Might be a problem. Do investigate logfile (${HOSTNAME}:${LOG})."|tee -a ${LOG}|mailx -s ":${HOSTNAME}: ERROR in NetApp-accounting for ${CLUSTER} [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed ERROR to ${MAILTO}"|tee -a ${LOG}
  fi

# copy file by using SCP
  scp ${DATADIR}/${CLUSTER}.asc ${SCPUSER}@${SCPHOST}:${SCPDIR}/${CLUSTER}-${DATE_YMD}.asc
  scp ${DATADIR}/${CLUSTER}.asc ${SCPUSER2}@${SCPHOST2}:${SCPDIR2}/${CLUSTER}-${DATE_YMD}.asc
  echo "`date` Information SCP-ed to ${SCPUSER}@${SCPHOST}:${SCPDIR} "|tee -a ${LOG}
  echo "`date` Information SCP-ed to ${SCPUSER2}@${SCPHOST2}:${SCPDIR2} "|tee -a ${LOG}
  ${SSH} ${SCPUSER}@${SCPHOST} ls -l ${SCPDIR}|tee -a ${LOG} 2>&1
  ${SSH} ${SCPUSER2}@${SCPHOST2} ls -l ${SCPDIR2}|tee -a ${LOG} 2>&1

done  # for CLUSTER

# Finish
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
rm ${TMP}
exit 0