#!/bin/sh
set +x
#--------------------------------------------------------------------------------#
# File    : cdot_upload_backup.sh                                                #
# By      : richard.loos@atos.net, 180912                                        #
# Subject : Script to upload backup files from cDOT                              #
# (1.0)   : 1.x was made by Richard Loos                                         #
#                                                                                #
#--------------------------------------------------------------------------------#
PGM="`basename $0|cut -d\. -f1`"
VER="1v0"
TMP="/tmp/${PGM}.$$"
HOSTNAME="`hostname|cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
CLUSTERS="${HOME}/etc/clusters"
ACCOUNT="`cat ${HOME}/etc/account`"
SERVER="10.192.109.45"
DIRECTORY="cdot"
ERR="${HOME}/log/${PGM}.err"
TMPERR="/tmp/${PGM}.$$.err"

#--------------------------------------------------------------------------------#
# SSH function                                                                   #
# 1: Filername 2:Command-string                                                  #
# When issue with connection to cluster, try the nodes (-01 & -02)               #
# "There are no entries matching your query." => EC=255                          #
# "no connection" is also EC=255                                                 #
#--------------------------------------------------------------------------------#
SSHCMD()
{
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    echo "  EC=${EC} EC2=${EC2} ${1} ${2} " >> ${ERR}
    sleep 2
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      echo "  EC=${EC} EC2=${EC2} ${1}-01 ${2} " >> ${ERR}
      sleep 2
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        echo "  EC=${EC} EC2=${EC2} ${1}-02 ${2} " >> ${ERR}
        sleep 2
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          echo "  EC=${EC} EC2=${EC2} ${1}-03 ${2} " >> ${ERR}
          echo "  EC=${EC} sleep 2 ..."
          sleep 2
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            echo "  EC=${EC} EC2=${EC2} ${1}-04 ${2} " >> ${ERR}
            echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -04 failed too."|tee -a ${LOG} ${ERR}
          fi
        fi
      fi
    fi
  fi
}

#--------------------------------------------------------------------------------#
# Upload backup file per node                                                    #
#--------------------------------------------------------------------------------#
for CLUSTER in `cat ${CLUSTERS}|grep -v \^#|awk -F\; '{print $1}'|grep "${FILTER}"`
do
  for NODE in `SSHCMD ${CLUSTER} 'node show -field node'|grep ${CLUSTER}|awk '{print $1}'|sort`
  do
    SSHCMD ${CLUSTER} 'set -privilege advanced;system configuration backup show '|grep ${NODE}|grep 8hour|tail -1|awk '{print $2}'|while read FILE
    do
      #--------------------------------------------------------------------------#
      # Create subdirectory if not exist                                         #
      #--------------------------------------------------------------------------#
      if [ ! -d "${HOME}/${DIRECTORY}/${NODE}" ]
      then
        mkdir ${HOME}/${DIRECTORY}/${NODE}
      fi
      #--------------------------------------------------------------------------#
      COMMAND=`echo "set -priv advanced;system configuration backup upload -node ${NODE} -backup ${FILE} -destination ftp://${ACCOUNT}@${SERVER}/${DIRECTORY}/${NODE}/"`
      SSHCMD ${CLUSTER} "${COMMAND}"
      #--------------------------------------------------------------------------#
      # Exit with returncode                                                     #
      #--------------------------------------------------------------------------#
      export RETURNCODE=$?
      if [ "${RETURNCODE}" != 0 -a "${RETURNCODE}" != 10 ]
      then
        exit 0
      else
        #------------------------------------------------------------------------#
        # Remove files older then 7 days                                        #
        #------------------------------------------------------------------------#
        find ${HOME}/${DIRECTORY}/${NODE}/*.7z -mtime +7 -exec rm {} ';'
      fi
      #--------------------------------------------------------------------------#
    done
  done
done