#!/bin/sh
set +x
#------------------------------------------------------------------------#
# File    : cdot_upload_backup.sh
# By      : richard.loos@atos.net, 180912
# Subject : Script to upload backup files from cDOT
# (1.1)   : 1.x was made by Richard Loos
# (1.2),MdB	: Updated with FTPDIR. Due to security problems in FTP-server. And some more updates
#(1.3),MdB	: Create 1 ERRFILE
#(1.4),MdB,220331	: VAR mod's for nlxfsd11
#(1.5),MdB,220407	: Added RCFILE & CLUSTERF (in stead of FILTER), FTPSERVER (in stead of SERVER)
#(1.6),221021,MdB	: Added a bit more LOGing
#
PGM="`basename $0|cut -d\. -f1`"
VER="1v6"
TMP="/tmp/${PGM}.$$"
HOSTNAME="`hostname|cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
CLUSTERS="${HOME}/etc/clusters"
ACCOUNT="`cat ${HOME}/etc/account`"
#FTPSERVER="10.192.109.45"
#DIRECTORY="cdot"
ERR="${HOME}/log/${PGM}.err"
ERR="/tmp/${PGM}.$$.err"
#BACKUPDIR="/appl/backup/cdot"
#BACKUPDIR="/appl/dfm/backup_cdot"
BACKUPDIR="/opt/netapp/backup_cdot"
CLUSTERF="[?]*"
FTPSERVER="10.192.109.46"
FTPDIR="tmp"
MAILTO="maarten.deboer@atos.net"
RCFILE="${HOME}/etc/${PGM}.rc"

#--------------------------------------------------------------------#
# File	: sshcmd.func
# VER="0.1", <node>-04
# VER="0.2", <node>-06
# VER="0.3", <node>-10
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.tmperr"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                sleep 1
                /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
                EC=${?}
                grep 'Connection refused' ${TMPERR}
                EC2=${?}
                if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                  sleep 1
                  /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
                  EC=${?}
                  grep 'Connection refused' ${TMPERR}
                  EC2=${?}
                  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                    sleep 1
                    /usr/bin/ssh -n ${1}-12 "${2}" 2> ${TMPERR}
                    EC=${?}
                    grep 'Connection refused' ${TMPERR}
                    EC2=${?}
                    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                      sleep 1
                      /usr/bin/ssh -n ${1}-13 "${2}" 2> ${TMPERR}
                      EC=${?}
                      grep 'Connection refused' ${TMPERR}
                      EC2=${?}
                      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                        echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 .. -10 failed too."|tee -a ${LOG} 
                      fi  # -13
                    fi  # -12
                  fi  # -11
                fi  # -10
              fi  # -09
            fi  # -08
          fi  # -07
        fi  # -06
      fi  # -05
    fi  # -01
  fi  # 
  rm ${TMPERR}
  return ${EC}
}

echo "`date` ${PGM} ${VER} started."|tee -a ${LOG}
if [ -f ${RCFILE} ]; then
  echo "  RCFILE (${RCFILE}) found. Using it ..."|tee -a ${LOG}
  . ${RCFILE}
fi
echo "  BACKUPDIR=${BACKUPDIR}"|tee -a ${LOG}
echo "  CLUSTERF=${CLUSTERF}"|tee -a ${LOG}
echo "  FTPSERVER=${FTPSERVER}"|tee -a ${LOG}
echo "  FTPDIR=${FTPDIR}"|tee -a ${LOG}
echo "  MAILTO=${MAILTO}"|tee -a ${LOG}

sleep 1

touch ${TMP} ${ERR}
#-----------------------------------------------------------------#
# Upload backup file per node                                     #
#-----------------------------------------------------------------#
for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${CLUSTERF}"`
do
# Cleanup 1st
  find ${HOME}/${FTPDIR}/${CLUSTER}*/*.7z -mtime +1 -exec rm {} ';'

  for NODE in `SSHCMD ${CLUSTER} 'node show -field node'|grep ${CLUSTER}|awk '{print $1}'|sort`
  do
    SSHCMD ${CLUSTER} 'set -privilege advanced;system configuration backup show '|grep ${NODE}|grep 8hour|tail -1|awk '{print $2}'|while read FILE
    do
      echo "  ${NODE} ..."|tee -a ${LOG}
      #-------------------------------------------------------------------#
      # Create subdirectory if not exist                                  #
      #-------------------------------------------------------------------#
      if [ ! -d "${BACKUPDIR}/${NODE}" ]
      then
        mkdir ${BACKUPDIR}/${NODE}
      fi
      if [ ! -d "${HOME}/${FTPDIR}/${NODE}" ]
      then
        mkdir ${HOME}/${FTPDIR}/${NODE}
      fi
      #--------------------------------------------------------------------#
      COMMAND=`echo "set -priv advanced;system configuration backup upload -node ${NODE} -backup ${FILE} -destination ftp://${ACCOUNT}@${FTPSERVER}/${FTPDIR}/${NODE}/"`
echo "  CMD=${COMMAND}"
      SSHCMD ${CLUSTER} "${COMMAND}" 1> ${TMP} 2>&1
      RC=${?}
echo "  RC=${RC}"
      if [ ${RC} -eq 0 ]; then
      # Mv the file from TEMP to backup-dest.dir, when FTP is OK (ReturnCode=0)
        echo "  Moving ${FILE} to ${BACKUPDIR}/${NODE}/ ..."
        mv ${HOME}/${FTPDIR}/${NODE}/${FILE} ${BACKUPDIR}/${NODE}/
      else
        cat ${TMP}
      fi  # RC=0
      #--------------------------------------------------------------------#
      # Exit with returncode                                               #
      #--------------------------------------------------------------------#
      export RETURNCODE=${RC}
      if [ "${RETURNCODE}" != 0 -a "${RETURNCODE}" != 10 ]
      then
        echo "  ERROR(${RETURNCODE}) in ${PGM} at ${HOSTNAME}"|tee -a ${LOG} ${ERR}
        cat ${TMP} >> ${ERR} 
        exit 0
      else
        #-------------------------------------------------------------------#
        # Remove files older then 7 days                                    #
        #-------------------------------------------------------------------#
echo "  Checking old files ..."
        find ${BACKUPDIR}/${NODE}/*.7z -mtime +7 -exec rm {} ';'
      fi
      #---------------------------------------------------------------------#
    done
  done
  # find ${BACKUPDIR}/${CLUSTER}/*.7z -mtime +1 -exec rm {} ';'
  # find ${HOME}/temp/${CLUSTER}/*.7z -mtime +0 -exec rm {} ';'
done  # CLUSTER

if [ -s ${ERR} ]; then
  echo "ERROR(s) in ${PGM} v${VER} at ${HOSTNAME}. See attachment. Disk usage = `df -g .`"| mailx -a ${ERR} -s ":${HOSTNAME}: ERROR(s) [${PGM} ${VER}]" ${MAILTO}
  echo "  Mailed ERRor (${ERR}) to ${MAILTO}"|tee -a ${LOG}
fi  # ERR

rm ${TMP} ${ERR}
echo "`date` ${PGM} ${VER} finished."|tee -a ${LOG}
exit 0