#!/bin/sh
# File	: cdot_backup_audit-logs.sh
# By	: Maarten.deBoer@Atos.net, 190103
# Subject	: Script to backup(collect) all (cDOT-)audit-log-files (per node) and put it in a directory structure
#(0.2),230922	: Re-newed
# Files are copied by scp -B
# For this (local) SSH-key need to be generated ("set -priv diag;system node systemshell -node nlnaf100-05 -command ssh-keygen")
# The .pub-key need to be placed into .ssh/authorized_keys
#
PGM="`basename $0|cut -d\. -f1`"
VER="0.2"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
MAILTO="maarten.deboer@atos.net"
HOSTNAME="`hostname`"
CLUSTERS="${HOME}/etc/clusters"
SSH="/usr/bin/ssh -n"

CFILTER="[?]*"
NFILTER="[?]*"

SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}

USAGE()
{
  echo "Usage: ${PGM} <options> "
  echo "  Version: ${VER}"
  echo "  options       :"
  echo "    -e          : Etc-file (${ETC})"
  echo "    -h          : this help"
  echo "    -x          : set -x"
  echo "    -V          : Version"
  echo "    --mailto    : do send mail (${MAILTO}) "
  echo "    --help      : this help"
}

# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -e) ETC=${2}; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

ACCOUNT="`cat ${HOME}/etc/account|cut -d\: -f1`"
FTPSERVER="10.192.109.46"
FTPDIR="tmp"
CFILTER="nlnaf100"
NFILTER="nlnaf100-05"


touch ${TMP}

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "  ACCOUNT=${ACCOUNT}"
echo "  FTPSERVER=${FTPSERVER}"
echo "  FTPDIR=${FTPDIR}"
echo "  CFILTER=${CFILTER}"
echo "  NFILTER=${NFILTER}"
sleep 1

touch ${TMP}
for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|grep "${CFILTER}"|sort`
do
  echo "${CLUSTER}:"
  SSHCMD ${CLUSTER} "node show -health true -field node"|grep ${CLUSTER}|grep "${NFILTER}"|while read NODE REST
  do
    echo "${CLUSTER};${NODE}"
    SSHCMD ${CLUSTER} "set diag; system node systemshell -node ${NODE} -command ls -1 /mroot/etc/log/mlog/audit.log.*"|grep "mroot"|while read FILE
    do
      echo "  FILE=${FILE}"
#      COMMAND=`echo "set -priv diag ; system node systemshell -node ${NODE} -command scp -B ${FILE} ${ACCOUNT}@${FTPSERVER}:/${FTPDIR}/${NODE}/"`
#      echo "  CMD=${COMMAND}"
#      SSHCMD ${CLUSTER} "${COMMAND}" 1> ${TMP} 2>&1
      ssh -n ${CLUSTER} "set -priv diag ; system node systemshell -node ${NODE} -command scp -B ${FILE} ${ACCOUNT}@${FTPSERVER}:/${FTPDIR}/${NODE}/" 
      RC=${?}
echo "  RC=${RC}"

    done  # LINE

# 
  done  # for NODE
done  # for CLUSTER

rm ${TMP}
exit 0