#!/bin/ksh
# File  : cdot_get_cluster_uptime.ksh
# By    : Maarten de Boer, 170821
# Subject       : Script to get cDOT uptime
# set -x
PGM="`basename $0|cut -d\. -f1`"
VER="0.1"
TMP="/tmp/${PGM}.$$"
CSV="${PGM}.csv"
CLUSTERS="${HOME}/etc/clusters"
MAILTO="maarten.deboer@atos.net"
MSGS="${TMP}.msgs"
SSH="/usr/bin/ssh"
HOSTNAME="`hostname | cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
ASC="${PGM}.asc"
FILTER="[?]*"
MAIL=""


SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.err"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
echo "1e EC=${EC} EC2=${EC2}"|tee -a ${LOG}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
echo "2e EC=${EC} EC2=${EC2}"|tee -a ${LOG}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-03 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-04 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 - -06 failed too."|tee -a ${LOG} 
              fi  # EC=0 & EC2=0
            fi  # -06
          fi  # -05
        fi  # -04
      fi  # -03
    fi  # -02
  fi  # -01
  rm ${TMPERR}
}


USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options       :"
  echo "    -e|--etc    : Etc/clusters-file (${CLUSTERS})"
  echo "    -f          : Filter filername (${FILTER})"
  echo "    -h|--help   : this Help"
  echo "    -m|--mail   : do send Mail"
  echo "    -s|--simple : Simple (without weeks) output"
  echo "    -V          : show Version"
  echo "    -x          : set -x"
  echo "    --mailto    : change MAILTO address & do send mail (${MAILTO})"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -e | --etc) FILERS=$2; shift ;;
    -f) FILTER=$2; shift ;;
    -m | --mail) MAIL=1 ;;
    --mailto) MAILTO=$2; MAIL=1; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;;
  esac
    shift
done  # case


echo "`date` ${PGM} v${VER} started"|tee -a ${LOG}
touch ${TMP} ${MSGS} /tmp/${CSV} /tmp/${ASC}
DAMOYR="`date +%d'-'%m'-'%Y`"
DATEHR="`date +%H`"

echo "" > ${TMP}
echo "No messages yet" > ${MSGS}

cat "${CLUSTERS}"|grep -v \^#|awk -F\; '{print $1}'|sort|grep "${FILTER}"|while read CLUSTER
do
  echo "  ${CLUSTER}; Getting uptime info ..."
# Getting the nodes 1st
  SSHCMD ${CLUSTER} "vserver show -type node"|grep ${CLUSTER}|awk '{print $1}' > ${TMP}.2
  cat ${TMP}.2|grep ${CLUSTER}|while read NODE
  do
# The 1st line (without LF) can be :
#   ssh: connect to host nlnaf100-01 port 22: Connection refused
# The 2nd line can be:
#   nlnaf100-04 205 days 02:54
# So need the check this out
#echo "1st SSHCMD:"
#SSHCMD ${CLUSTER} "system node show -node ${NODE} -fields uptime"|grep ${CLUSTER}
    UPTIME=`SSHCMD ${CLUSTER} "system node show -node ${NODE} -fields uptime"|grep ${CLUSTER}`
    LOOP=1
    while [ ${LOOP} -eq 1 ]
    do
      LOOP=0
      UPTIDAYS=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $2}'|sed -e 's/,//g'`
      UPTITIME=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $4}'|sed -e 's/,//g'`
      if [ "${UPTIDAYS}" = "connect" ]; then
        UPTIDAYS=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $11}'|sed -e 's/,//g'`
        UPTITIME=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $13}'|sed -e 's/,//g'`
        if [ "${UPTIDAYS}" = "connect" ]; then
          UPTIDAYS=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $20}'|sed -e 's/,//g'`
          UPTITIME=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $22}'|sed -e 's/,//g'`
          if [ "${UPTIDAYS}" = "connect" ]; then
            UPTIDAYS=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $29}'|sed -e 's/,//g'`
            UPTITIME=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $31}'|sed -e 's/,//g'`
            if [ "${UPTIDAYS}" = "connect" ]; then
              UPTIDAYS=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $38}'|sed -e 's/,//g'`
              UPTITIME=`echo ${UPTIME}|grep ${CLUSTER}|awk '{print $40}'|sed -e 's/,//g'`
            fi  # 38
          fi  # 29
        fi  # 20
      fi # 11
      if [ "${UPTIDAYS}" = "connect" ]; then
        LOOP=1
        sleep 1
      fi  # connect
    done  # CHECK

    HR=`echo ${UPTITIME}|cut -d\: -f1|sed -e 's/ //g'`
    let HOURS="(${UPTIDAYS} * 24) + ${HR}"
    echo "${NODE};${HOURS}; ; ; ; ;${UPTIDAYS} days;${UPTITIME};;" | tee -a ${TMP}
  done  # cat ${TMP}
done  # CLUSTER

if [ ${MAIL} ]; then
  echo ";" | tee -a ${TMP}
  echo "# Filername ; Hours ; ; ; ; ; Days ; Uptime string; " | tee -a ${TMP}

  cp ${TMP} /tmp/${CSV}
  echo "`date`. Options; FILTER=${FILTER},MAIL=${MAIL}"|mailx -a /tmp/${CSV} -s ":${HOSTNAME}: Uptime-sheet [${PGM} v${VER}]" ${MAILTO}
  cp ${MSGS} /tmp/${ASC}
  echo "`date`. Options; FILTER=${FILTER},MAIL=${MAIL}"|mailx -a /tmp/${ASC} -s ":${HOSTNAME}: Messages-files [${PGM} v${VER}]" ${MAILTO}
  echo "`date` mailed to ${MAILTO} "|tee -a ${LOG}
fi  # MAIL

rm ${TMP} ${TMP}.2 ${MSGS} /tmp/${CSV} /tmp/${ASC}
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0