#!/bin/sh
# File	: chk_linux_servers.sh
# By	: Maarten.deBoer@atos.net, 240516
# Subject	: Script to check Linux systems (disk usage, etc. Also remote)
#(0.2),240516	: Mod's, WARN
#(0.3),240516	: Add EXCL
#(0.4),240528	: Added IMI2ESB
PGM=`basename $0|cut -d\. -f1`
VER="0.4"
TMP="/tmp/${PGM}.$$"
LOG="${HOME}/log/${PGM}.log"
ASC="/tmp/${PGM}.asc"
SSH="/usr/bin/ssh -n"
FILTER="[?]*"
#FILTER="nlxfsd19"
EXCL="${HOME}/etc/${PGM}.excl"
HOSTNAME=`hostname | cut -d\. -f1`

IMI2ESB_UPL="${HOME}/imi2esb/bin/imi_upload.sh"
#IMI2ESB_UPL=""
IMI2ESB_Q="${HOME}/imi2esb/queue/"
IMI2ESB_USER="sdp"
IMI2ESB_TXT="${TMP}.esb-txt"
W_SEC=30
IMI2ESB_REQBY="maarten.deboer@atos.net"
ESBSTAT_URL="https://btn.sapart.atos-srv.net/ART/esbstat.pl?action=info"

MAILTO="maarten.deboer@atos.net"
SYSTEMS="${HOME}/etc/system.hosts"
MAX_PERC=85

SEND_TO_ESB()
# $1=CI, $2=TYPE, $3=text
{
  EXT_REF="${IMI2ESB_USER}_`date +%Y%m%d%H%M%S`"
  TICKET="${IMI2ESB_Q}/${EXT_REF}.imi"
  echo "@PROXYUSER:${IMI2ESB_USER}" > ${TICKET}
  echo "@EXTERNALREFERENCE:${EXT_REF}" >> ${TICKET}
  echo "@REQUESTTYPE:${2}" >> ${TICKET}
  echo "@SEVERITY:3" >> ${TICKET}
  echo "@REQUESTAREA:${REQ_AREA}" >> ${TICKET}
  echo "@GROUP:${GROUP}" >> ${TICKET}
  echo "@ORGANISATION:${ORG}" >> ${TICKET}
  echo "@REQUESTEDBY:${IMI2ESB_REQBY}" >> ${TICKET}
  echo "@CI:${1}" >> ${TICKET}
  cat ${3} >> ${TICKET}
  echo "" >> ${TICKET}
  echo "" >> ${TICKET}
  echo "externalreference=${EXT_REF}" >> ${TICKET}
  echo "" >> ${TICKET}
  echo "@ENDOFUSDMESSAGE:" >> ${TICKET}
  echo "  Created IMI2ESB file ${TICKET}"|tee -a ${LOG}
  cp ${TICKET} ${TMP}.ticket
  sleep 1
  echo "  Sending to ESB"|tee -a ${LOG}
  ${IMI2ESB_UPL}
  echo "  Waiting (${W_SEC}sec.) for ticket info ..."
  sleep ${W_SEC}
#  wget -q --no-proxy --no-check-certificate -O ${TMP}.wget ${ESBSTAT_URL}
#  wget -q --no-check-certificate -O ${TMP}.wget ${ESBSTAT_URL}
  wget -q --no-proxy --no-check-certificate -O ${TMP}.wget ${ESBSTAT_URL}
  TICKET_INFO=`grep ${EXT_REF} ${TMP}.wget`
  echo "  Ticket info return: ${TICKET_INFO}"|tee -a ${LOG} ${TMP}
  rm ${TMP}.wget

  if [ "${MAILTO}" != "" ]; then
    cat ${TMP}.ticket| mailx -s ":${HOSTNAME}: ESB (ticket-)message ${TICKET_INFO} [${PGM} v${VER}]" ${MAILTO}
  fi
  rm ${TMP}.ticket
}  # SEND_TO_ESB

echo "`date` ${PGM} v${VER} started."|tee -a ${LOG}
echo "  EXCL=${EXCL}"
echo "  FILTER=${FILTER}"
echo "  MAILTO=${MAILTO}"
echo "  MAX_PERC=${MAX_PERC}"
echo "  SYSTEMS=${SYSTEMS}"
echo "  IMI2ESB_UPL=${IMI2ESB_UPL}"
echo "  IMI2ESB_Q=${IMI2ESB_Q}"
echo "  IMI2ESB_USER=${IMI2ESB_USER}"
echo "  IMI2ESB_TXT=${IMI2ESB_TXT}"
sleep 1

if [ ! -f ${SYSTEMS} ]; then
  echo "  NO SYSTEMS (${SYSTEMS}) found. Exiting ..."|tee -a ${LOG}
  exit 4
fi
if [ ! -f ${EXCL} ]; then
  touch ${EXCL}
fi  # EXCL

touch ${TMP} ${TMP}.esb-txt
cat ${SYSTEMS}|grep -v ^#|grep "${FILTER}"| while read LINE
do
  echo "-----"
  cp /dev/null ${TMP}.esb-txt
  IP_ADDR=`echo ${LINE}|awk '{print $1}'`
  HOST=`echo ${LINE}|awk '{print $2}'`
  echo "  IP_ADDR=${IP_ADDR} HOST=${HOST}"

# Check hostname
  HOST_NAME=`${SSH} ${IP_ADDR} "hostname"|cut -d\. -f1 2>/dev/null`
  echo "  HOST_NAME(${HOST_NAME}) HOST(${HOST}) "
  if [ "${HOST_NAME}" != "${HOST}" ]; then
    echo "  WARN: HOST_NAME(IP_ADDR) ${HOST_NAME} <> HOST(${HOST}) "|tee -a ${LOG} ${TMP}
  fi  # !=

# Check disk
  ${SSH} ${IP_ADDR} "mount"|grep ^/dev|awk '{print $1}'|grep -vf ${EXCL}|while read REM_DEV
  do
    echo "IP_ADDR=${IP_ADDR}(${HOST_NAME}) REM_DEV=${REM_DEV}"
# Remove "Filesystem", select on % and remove /dev/sd...
    DF_LINE=`${SSH} ${IP_ADDR} "df ${REM_DEV}"|grep -v "Filesystem"|grep '%'|sed 's|^/dev/sd[a-z][0-9]||g' 2>/dev/null `

    PERC=`echo "${DF_LINE}"|awk '{print $5}'|grep '%'|sed 's/%//g' `
    MNT_ON=`echo "${DF_LINE}"|awk '{print $6}'`
# Some older layout has shifted col's
    if [ "${MNT_ON}" = "" ]; then
      PERC=`echo "${DF_LINE}"|awk '{print $4}'|grep '%'|sed 's/%//g' `
      MNT_ON=`echo "${DF_LINE}"|awk '{print $5}'`
    fi

    if [ ${PERC} -ge ${MAX_PERC} ]; then
      echo "  WARNing: ${HOST}(${IP_ADDR}):${MNT_ON} ${PERC}(>${MAX_PERC})%"|tee -a ${LOG} ${TMP} ${TMP}.esb-txt
      echo "  Please fix. KR, ${MAILTO}" |tee -a ${TMP}.esb-txt
#      echo "DF_LINE=${DF_LINE}" >> ${TMP}
    fi  # > MAX_PERC    

# Create ticket
    if [ "${IMI2ESB_UPL}" != "" ] && [ -s ${TMP}.esb-txt ]; then
      echo "----- TICKET -----"
      cat ${TMP}.esb-txt
      sleep 1

#     ORG="Atos Netherlands"
#     REQ_AREA="Storage;Storage;Other"
#     GROUP="RO.Storage.FSOD"

      ORG="Atos Cloud CIS3"
      REQ_AREA="ServerMgmt;Generic;unspecified"
      GROUP="IN.Cloud.CIS3-Linux"

#     SEND_TO_ESB <CI> <TYPE> <MSG>
# $1=CI, $2=TYPE, $3=text
      SEND_TO_ESB ${HOST_NAME}-VSR "Incident" ${TMP}.esb-txt

      echo "`date` Uploaded to ${IMI2ESB_Q}"|tee -a ${LOG} ${TMP}
    fi  # IMI2ESB_UPL

  done  # DEVICE

done  # LINE

if [ "${MAILTO}" != "" ] && [ -s ${TMP} ]; then
  cp ${TMP} ${ASC}
  echo "`date` ----- ${PGM} v${VER} @${HOSTNAME} -----" >> ${ASC}
  WARN_CNT=`cat ${ASC}|grep -c "WARN:"`
  echo "${WARN_CNT} WARN's. See attachement for details" | mailx -a ${ASC} -s ":${HOSTNAME}: Check of Linux systems [${PGM} v${VER}]" "${MAILTO}"
  echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
fi  # MAILTO

rm ${TMP} ${TMP}.esb-txt
echo "`date` ${PGM} v${VER} finished."|tee -a ${LOG}
exit 0