#!/bin/sh
# File	: cdot_chk-set_ntp-time.sh
# By	: Maarten.deBoer@atos.net, 221121
# Subject	: Script to chk (NTP-)time
#(0.2),221121	: Added USAGE(), SET_TIME, chk -> chk-set
#(0.3),230227	: Added SSHCMD()
#set -x
PGM=`basename $0|cut -d\. -f1`
VER="0.3"
TMP="/tmp/${PGM}.$$"
ASC="/tmp/${PGM}.$$.asc"
INCIFILE="/tmp/${PGM}.$$.inci"
LOG="${HOME}/log/${PGM}.log"
SSH="/usr/bin/ssh -n"
CLUSTERS="${HOME}/etc/clusters"
HOSTNAME=`hostname |cut -d\. -f1`
MAIL=""
MAILTO="maarten.deboer@atos.net"
CFILTER="[?]*"
SET_TIME=""

INCI=""
ESB=""
ORG="Atos Netherlands"
REQ_TYPE="Incident"
REQ_AREA="Storage;Storage;Other"
GROUP="NL.Storage.FSOD"
#GROUP="RO.Storage.FSOD"
ESB_QUEUE="${HOME}/imi2esb/queue"
ESB_USER="sdp"
IMI_UPLOAD="${HOME}/imi2esb/bin/imi_upload.sh"
#ESBSTAT_URL="https://btndev.sapart.atos-srv.net/ART/esbstat.pl?action=info"
ESBSTAT_URL="https://btn.sapart.atos-srv.net/ART/esbstat.pl?action=info"
W_SEC=30


# File	: sshcmd.func
# VER="0.1", <node>-04
# VER="0.2", <node>-06
# VER="0.3", <node>-10
SSHCMD()
# 1: Filername 2:Command-string
# When issue with connection to cluster, try the nodes (-01 & -02)
# "There are no entries matching your query." => EC=255
# "no connection" is also EC=255
{
  TMPERR="/tmp/${PGM}.$$.tmperr"
  touch ${TMPERR}
  /usr/bin/ssh -n ${1} "${2}" 2> ${TMPERR}
  EC=${?}
  # Check if "ssh: connect to host 10.192.109.202 port 22: Connection refused" If so (EC2=0), the 2nd
  grep 'Connection refused' ${TMPERR}
  EC2=${?}
  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
    sleep 1
    /usr/bin/ssh -n ${1}-05 "${2}" 2> ${TMPERR}
    EC=${?}
    grep 'Connection refused' ${TMPERR}
    EC2=${?}
    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
      sleep 1
      /usr/bin/ssh -n ${1}-01 "${2}" 2> ${TMPERR}
      EC=${?}
      grep 'Connection refused' ${TMPERR}
      EC2=${?}
      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
        sleep 1
        /usr/bin/ssh -n ${1}-06 "${2}" 2> ${TMPERR}
        EC=${?}
        grep 'Connection refused' ${TMPERR}
        EC2=${?}
        if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
          sleep 1
          /usr/bin/ssh -n ${1}-02 "${2}" 2> ${TMPERR}
          EC=${?}
          grep 'Connection refused' ${TMPERR}
          EC2=${?}
          if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
            sleep 1
            /usr/bin/ssh -n ${1}-07 "${2}" 2> ${TMPERR}
            EC=${?}
            grep 'Connection refused' ${TMPERR}
            EC2=${?}
            if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
              sleep 1
              /usr/bin/ssh -n ${1}-08 "${2}" 2> ${TMPERR}
              EC=${?}
              grep 'Connection refused' ${TMPERR}
              EC2=${?}
              if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                sleep 1
                /usr/bin/ssh -n ${1}-09 "${2}" 2> ${TMPERR}
                EC=${?}
                grep 'Connection refused' ${TMPERR}
                EC2=${?}
                if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                  sleep 1
                  /usr/bin/ssh -n ${1}-10 "${2}" 2> ${TMPERR}
                  EC=${?}
                  grep 'Connection refused' ${TMPERR}
                  EC2=${?}
                  if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                    sleep 1
                    /usr/bin/ssh -n ${1}-11 "${2}" 2> ${TMPERR}
                    EC=${?}
                    grep 'Connection refused' ${TMPERR}
                    EC2=${?}
                    if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                      sleep 1
                      /usr/bin/ssh -n ${1}-12 "${2}" 2> ${TMPERR}
                      EC=${?}
                      grep 'Connection refused' ${TMPERR}
                      EC2=${?}
                      if [ ${EC} -ne 0 ] && [ ${EC2} -eq 0 ]; then
                        echo  "`date` ${PGM} ERROR with communication to ${1}. Connection to -01 .. -10 failed too."|tee -a ${LOG} 
                      fi 
                    fi
                  fi 
                fi 
              fi 
            fi  
          fi  
        fi  
      fi  
    fi  
  fi  # 
  rm ${TMPERR}
}

SEND_TO_ESB()
# $1=CI, $2=TYPE, $3=text
{
  EXT_REF="${ESB_USER}_`date +%Y%m%d%H%M%S`"
  TICKET="${ESB_QUEUE}/${EXT_REF}.imi"
  echo "@PROXYUSER:${ESB_USER}" > ${TICKET}
  echo "@EXTERNALREFERENCE:${EXT_REF}" >> ${TICKET}
  echo "@REQUESTTYPE:${2}" >> ${TICKET}
  echo "@SEVERITY:3" >> ${TICKET}
  echo "@REQUESTAREA:${REQ_AREA}" >> ${TICKET}
  echo "@GROUP:${GROUP}" >> ${TICKET}
  echo "@ORGANISATION:${ORG}" >> ${TICKET}
  echo "@CI:${1}" >> ${TICKET}
  cat ${3} >> ${TICKET}
  echo "" >> ${TICKET}
  echo "" >> ${TICKET}
  echo "externalreference=${EXT_REF}" >> ${TICKET}
  echo "" >> ${TICKET}
  echo "@ENDOFUSDMESSAGE:" >> ${TICKET}
  echo "  Created IMI file ${TICKET}"|tee -a ${LOG}
  cp ${TICKET} ${TMP}.ticket
  sleep 1
  echo "  Sending to ESB"|tee -a ${LOG} ${TMP}
  ${IMI_UPLOAD}
  echo "  Waiting (${W_SEC}sec.) for ticket info ..."
  sleep ${W_SEC}
#  wget -q --no-proxy --no-check-certificate -O ${TMP}.func ${ESBSTAT_URL}
  wget -q --no-check-certificate -O ${TMP}.func ${ESBSTAT_URL}
  TICKET_INFO=`grep ${EXT_REF} ${TMP}.func`
  echo "  Ticket info return: ${TICKET_INFO}"|tee -a ${LOG} ${TMP}
  rm ${TMP}.func
  
  if [ ${MAIL} ]; then
    cat ${TMP}.ticket| mailx -s ":${HOSTNAME}: ESB (ticket-)message ${TICKET_INFO} [${PGM} v${VER}]" ${MAILTO}
  fi
  rm ${TMP}.ticket
}  # ESB

USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options       :"
  echo "    -c          : Cluster-filter (${CFILTER})"
  echo "    -m          : Mail output"
  echo "    --mailto    : set MAILTO (${MAILTO})"
  echo "    --set-time  : SET-TIME"
  echo "    -h|--help   : this help"
  echo "    -V          : Version"
  echo "    -x          : set -x"
}
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -i|--inci) INCI=1 ;;
    -m) MAIL=1 ;;
    -c) CFILFTER="${2}"; shift ;;
    --set-time) SET_TIME=1; shift ;;
    --mailto) MAILTO="${2}"; MAIL=1; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

echo "`date` ${PGM} v${VER} started at `hostname`."|tee -a ${LOG} ${TMP}.mail
echo "  CLUSTERS=${CLUSTERS}"|tee -a ${TMP}.mail
echo "  CFILFTER=${CFILFTER}"|tee -a ${TMP}.mail
echo "  INCI=${INCI}"|tee -a ${TMP}.mail
echo "  MAIL=${MAIL}"|tee -a ${TMP}.mail
echo "  MAILTO=${MAILTO}"|tee -a ${TMP}.mail
echo "  SET_TIME=${SET_TIME}"|tee -a ${TMP}.mail

touch ${TMP} ${INCIFILE} ${TMP}.mail
for CLUSTER in `cat ${CLUSTERS}|grep -v ^#|grep "${CFILTER}"`
do
  echo "${CLUSTER} ..."|tee ${TMP}

# date +%a' '%b' '%d' '%H:%M:%S' '%Y
  CDATE=`SSHCMD ${CLUSTER} "date -u"|grep ${CLUSTER}| awk '{print $2,$3,$4,$5,$6}'|head -1`
  SDATE=`date -u +%a' '%b' '%d' '%H:%M:%S' '%Y`
  echo "  System-date=${SDATE}, Cluster-date=${CDATE}"|tee -a ${TMP}
  ANSW=`echo ${CDATE}|grep "${SDATE}"`
  if [ "${ANSW}" = "" ]; then
# Not the same time. Check again
    CDATE=`SSHCMD ${CLUSTER} "date -u"|grep ${CLUSTER}| awk '{print $2,$3,$4,$5,$6}'|head -1`
    SDATE=`date -u +%a' '%b' '%d' '%H:%M:%S' '%Y`
    echo "  2nd System-date=${SDATE}, Cluster-date=${CDATE}" |tee -a ${TMP}
    ANSW2=`echo ${CDATE}|grep "${SDATE}"`
    if [ "${ANSW2}" = "" ]; then
      INCI_TXT="Management (${HOSTNAME}) System-date & Cluster-date (${CLUSTER}) should be the same. Are now ${SDATE} vs ${CDATE}."
      echo "${CLUSTER} => ${INCI_TXT} " > ${INCIFILE}
      echo "${CLUSTER} => ${INCI_TXT} "|tee -a ${TMP} ${ASC}

      if [ ${INCI} ]; then
#set -x
        SEND_TO_ESB "${CLUSTER}-PLF" "Incident" ${INCIFILE}
#set +x
      fi  # INCI

      if [ ${SET_TIME} ]; then
        echo "  SETting TIME ${CLUSTER} ..."|tee -a ${LOG}
# echo "# [ -u|-u <[[[[[cc]yy]mm]dd]hhmm[.ss]]> ]  UTC Date and Time"
        DATI=`date +%Y%m%d%H%M.%S`
        echo ${DATI}
        SSHCMD ${CLUSTER} "date" |tee -a ${LOG}
        SSHCMD ${CLUSTER} "date ${DATI}"
        SSHCMD ${CLUSTER} "date" |tee -a ${LOG}

      fi  # SET_TIME

    fi  # ANSW2
  fi  # ANSW


done  # for CLUSTER


if [ ${MAIL} ] && [ "${MAILTO}" != "" ] && [ -s ${ASC} ]; then
  clear
  echo "=========================================================="
  cat ${ASC}
  cat ${TMP}.mail |mailx -a ${ASC} -s ":${HOSTNAME}: cDOT time ERRORs [${PGM} v${VER}]" ${MAILTO}
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  rm ${ASC}
fi

rm ${TMP} ${INCIFILE} ${TMP}.mail
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0