#!/bin/sh
set +x
#--------------------------------------------------------------------------------#
# File    : cdot_upload_backup.sh                                                #
# By      : richard.loos@atos.net, 180912                                        #
# Subject : Script to upload backup files from cDOT                              #
# (1.1)   : 1.x was made by Richard Loos                                         #
# (1.2),MdB	: Updated with FTPDIR. Due to security problems in FTP-server. And some more updates
#(1.3),MdB	: Create 1 ERRFILE
#                                                                                #
#--------------------------------------------------------------------------------#
PGM="`basename $0|cut -d\. -f1`"
VER="1v3"
TMP="/tmp/${PGM}.$$"
HOSTNAME="`hostname|cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
CLUSTERS="${HOME}/etc/clusters"
ACCOUNT="`cat ${HOME}/etc/account`"
SERVER="10.192.109.45"
#DIRECTORY="cdot"
FTPDIR="temp"
ERR="${HOME}/log/${PGM}.err"
FILTER="[?]*"
#FILTER="nlnaf106"
ERR="/tmp/${PGM}.$$.err"
BACKUPDIR="/appl/backup/cdot"
MAILTO="richard.loos@atos.net maarten.deboer@atos.net"

#--------------------------------------------------------------------------------#
# File	: sshcmd.func
# VER="0.1", <node>-04
# VER="0.2", <node>-06
# VER="0.3", <node>-10
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.tmperr"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                sleep 1
                /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
                EC=${?}
                grep 'Connection refused' ${TMPERR}
                EC2=${?}
                if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                  sleep 1
                  /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
                  EC=${?}
                  grep 'Connection refused' ${TMPERR}
                  EC2=${?}
                  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                    sleep 1
                    /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
                    EC=${?}
                    grep 'Connection refused' ${TMPERR}
                    EC2=${?}
                    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                      sleep 1
                      /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
                      EC=${?}
                      grep 'Connection refused' ${TMPERR}
                      EC2=${?}
                      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                        echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 .. -10 failed too."|tee -a ${LOG} 
                      fi  # -10
                    fi  # -09
                  fi  # -08
                fi  # -07
              fi  # -06
            fi  # -05
          fi  # -04
        fi  # -03
      fi  # -02
    fi  # -01
  fi  # 
  rm ${TMPERR}
  return ${EC}
}

echo "`date` ${PGM} ${VER} started."|tee -a ${LOG}
touch ${TMP} ${ERR}
#--------------------------------------------------------------------------------#
# Upload backup file per node                                                    #
#--------------------------------------------------------------------------------#
for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${FILTER}"`
do
  for NODE in `SSHCMD ${CLUSTER} 'node show -field node'|grep ${CLUSTER}|awk '{print $1}'|sort`
  do
    SSHCMD ${CLUSTER} 'set -privilege advanced;system configuration backup show '|grep ${NODE}|grep 8hour|tail -1|awk '{print $2}'|while read FILE
    do
      #--------------------------------------------------------------------------#
      # Create subdirectory if not exist                                         #
      #--------------------------------------------------------------------------#
      if [ ! -d "${BACKUPDIR}/${NODE}" ]
      then
        mkdir ${BACKUPDIR}/${NODE}
      fi
      if [ ! -d "${HOME}/${FTPDIR}/${NODE}" ]
      then
        mkdir ${HOME}/${FTPDIR}/${NODE}
      fi
      #--------------------------------------------------------------------------#
      COMMAND=`echo "set -priv advanced;system configuration backup upload -node ${NODE} -backup ${FILE} -destination ftp://${ACCOUNT}@${SERVER}/${FTPDIR}/${NODE}/"`
echo "CMD=${COMMAND}"
      SSHCMD ${CLUSTER} "${COMMAND}" 1> ${TMP} 2>&1
      RC=${?}
echo "RC=${RC}"
      if [ ${RC} -eq 0 ]; then
      # Mv the file from TEMP to backup-dest.dir, when FTP is OK (ReturnCode=0)
        echo "  Moving ${FILE} to ${BACKUPDIR}/${NODE}/ ..."
        mv ${HOME}/${FTPDIR}/${NODE}/${FILE} ${BACKUPDIR}/${NODE}/
      else
        cat ${TMP}
      fi  # RC=0
      #--------------------------------------------------------------------------#
      # Exit with returncode                                                     #
      #--------------------------------------------------------------------------#
      export RETURNCODE=${RC}
      if [ "${RETURNCODE}" != 0 -a "${RETURNCODE}" != 10 ]
      then
        echo "  ERROR(${RETURNCODE}) in ${PGM} at ${HOSTNAME}"|tee -a ${LOG} ${ERR}
        cat ${TMP} >> ${ERR} 
        exit 0
      else
        #------------------------------------------------------------------------#
        # Remove files older then 7 days                                        #
        #------------------------------------------------------------------------#
echo "  Checking old files ..."
        find ${BACKUPDIR}/${NODE}/*.7z -mtime +7 -exec rm {} ';'
      fi
      #--------------------------------------------------------------------------#
    done
  done
##  find ${BACKUPDIR}/${CLUSTER}/*.7z -mtime +7 -exec rm {} ';'
done  # CLUSTER

if [ -s ${ERR} ]; then
  echo "ERROR(s) in ${PGM} v${VER} at ${HOSTNAME}. See attachment. Disk usage = `df -g .`"| mailx -a ${ERR} -s ":${HOSTNAME}: ERROR(s) [${PGM} ${VER}]" ${MAILTO}
fi  # ERR

rm ${TMP} ${ERR}
echo "`date` ${PGM} ${VER} finished."|tee -a ${LOG}
exit 0