#!/bin/sh
# File	: cdot_backup_audit-logs.sh
# By	: Maarten.deBoer@Atos.net, 190103
# Subject	: Script to backup(collect) all (cDOT-)audit-log-files (per node) and put it in a directory structure
PGM="`basename $0|cut -d\. -f1`"
VER="0.1"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
MAILTO="maarten.deboer@atos.net"
HOSTNAME="`hostname`"
CLUSTERS="${HOME}/etc/clusters"
SSH="/usr/bin/ssh -n"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

USAGE()
{
  echo "Usage: ${PGM} <options> "
  echo "  Version: ${VER}"
  echo "  options       :"
  echo "    -e          : Etc-file (${ETC})"
  echo "    -h          : this help"
  echo "    -x          : set -x"
  echo "    -V          : Version"
  echo "    --mailto    : do send mail (${MAILTO}) "
  echo "    --help      : this help"
}

# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -e) ETC=${2}; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

touch ${TMP}

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}


for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|sort`
do
  echo "${CLUSTER}:"
  SSHCMD ${CLUSTER} "node show -health true -field node"|grep ${CLUSTER}|while read NODE REST
  do
    echo "${CLUSTER};${NODE}"
    SSHCMD ${CLUSTER} "set diag; system node systemshell -node ${NODE} -command ls -l /mroot/etc/log/mlog/audit.log.*"
# 
# http://docs.netapp.com/ontap-9/index.jsp?topic=%2Fcom.netapp.doc.dot-cm-sag%2FGUID-279ACA3C-00D2-490C-BEE9-C05625A550B1.html
# 
# Forwarding the audit log to a destination
# You can forward the audit log to a maximum of 10 destinations that you specify by using the cluster log-forwarding create command. 
# For example, you can forward the log to a Splunk or syslog server for monitoring, analysis, or backup purposes.
# 
#
  done  # for NODE
done  # for CLUSTER

rm ${TMP}
exit 0