#!/bin/ksh
set +x
#--------------------------------------------------------------------------#
# Name: check_old_snaps.sh                                                 #
# Description:  Script checks old snapshots from snapmirror, snapvault,    #
#               etc.                                                       #
#                                                                          #
# Usage:  check_old_snaps.sh [options]                                     #
#                                                                          #
# Author(s): Atos                                                          #
#                                                                          #
# Change log:                                                              #
# aut   date            vers    comments                                   #
# RL    03-06-2010      1.0     Initial version Richard Loos               #
# MdB	13-04-2011	2.0	Added ticket generation, options,          #
#                               vfiler running                             #
# MdB	26-05-2011	2.1	Added to SDM                               #                
# MdB	21-10-2014	2.2	Mod SDM->SDM & SDMMAIL, Incident -> Query; if TMPFILE not empty; mail.
# MdB	10-03-2016	2.3	Mod license for 8.2
# MdB	07-09-2016	2.4	Added excludes (for SLB)
# MdB   13-10-2016      2.5     Added option -e 
# MdB	15-03-2017	2.6	Added -p|--(nagios)passive(check)
#--------------------------------------------------------------------------#
# Initialize variabels                                                     #
#--------------------------------------------------------------------------#
PGM="`basename $0|cut -d\. -f1`"
VER="2.5"
HOSTNAME="`hostname|cut -d\. -f1`"
LOG="${HOME}/log/${PGM}.log"
LOGDIR="${HOME}/log"
#TMPDIR=${HOME}/data/tmp
TMPDIR="/tmp"
RUNDATE=$(date +%a)
LOGFILE="${LOGDIR}/${PGM}_${RUNDATE}.log"
TMPFILE="${TMPDIR}/${PGM}.tmp.$$"
DOEXCL=""
EXCL="${HOME}/etc/${PGM}.exclude"
# !! -n at ssh is NEEDED. Otherwise loop wil stop after 1st run  #
SSH="ssh -n"
# ERR is a (tmp)file with ERRors in it. Used for SDM / Nagios-Passivecheck
ERR="/tmp/${PGM}.$$.err"
MAIL=""
MAILTO="fsod@atos.net"
FILTER="[?]*"
FILERS="${HOME}/etc/filers"

SDM=""
#SDMMAIL="email@usd-prod.uk.atosorigin.com"
#SDMMAIL="prod-imi@gis.nl.ao-srv.com"
SDMMAIL="ESBIncident.it-solutions@atos.net"
#SDMUSER="nl19471"
SDMUSER="nldsm01"
SDMMSG="${TMP}.usd"
SDMREPLYTO="fsod@atos.net"

PASSIVE=""
#PASSIVECHKDIR="/appl/dfm/nagios/PassiveCheck/PRD/"
PASSIVECHKDIR="/appl/dfm/nagios/PassiveCheck/DVL/"
MONIDHDR="MAS.NL.1"
CLASS="CLASS=ZZ-Event.Storage.Storage"

# if not me (accourding to LOGNAME), then change some user related info
if [ "${LOGNAME}" != "nl19471" ]; then
  MAILTO="fsod@atos.net"
  SDMUSER="nldsm01"
  SDMREPLYTO="fsod@atos.net"
fi

#--------------------------------------------------------------------------#
# Functions                                                                #
#--------------------------------------------------------------------------#
USAGE()
{
  echo "Usage: ${PGM} [<options>]"
  echo "  Version: ${VER}"
  echo "  options    :"
  echo "    -e | --etc     : Etc/filers-file (${FILERS})"
  echo "    -f             : filter filername (${FILTER})"
  echo "    -h | --help    : this help"
  echo "    -m | --mail    : do send mail"
  echo "    -p | --passive : using nacl(nagios-client)-PassiveCheck"
  echo "    -s | --sdm     : do send to SDM (via IMI to:${SDMMAIL})"
  echo "    -V             : Version"
  echo "    -x             : set -x"
  echo "    --excl         : do use EXCLude-file (${EXCL})"
  echo "    --mailto       : change MAILTO address & do send mail (${MAILTO})"
}
CREATE_SDM_TICKET()
# $1=Filer(CI), $2=SDMTYPE, $3=SDM-text
{
  echo "@REQUESTTYPE: ${2}" > ${SDMMSG}
  echo "@REQUESTAREA: NL.Storage.StorageOnDemand" >> ${SDMMSG}
  echo "@SEVERITY: 4" >> ${SDMMSG}
  echo "@CI: ${1}" >> ${SDMMSG}
  echo "@ORGANISATION: AtosOrigin.Netherlands" >> ${SDMMSG}
  echo "@GROUP: RO.Storage.FSOD" >> ${SDMMSG}
  echo "@REPLYTO: ${SDMREPLYTO}" >> ${SDMMSG}
  echo "@PROXYUSER: ${SDMUSER}"  >> ${SDMMSG}
  cat ${3} >> ${SDMMSG}
  echo "[${PGM} v${VER}] @${HOSTNAME}" >> ${SDMMSG}
  echo "@ENDOFSDMMESSAGE" >> ${SDMMSG}

  cat ${SDMMSG} | mailx -s "${SDMUSER}: WARNING; ${PGM} at ${1}." ${SDMMAIL}

  echo "`date` ${PGM}: Mailed to IMI about ${1} (${SDMMAIL})" | tee -a ${LOG}
}
## MAIN
# Check options
while [ $# -gt 0 ]
  do
  case $1 in
    -e | --etc) FILERS=$2; shift ;;
    -f) FILTER=$2; shift ;;
    -h | --help) USAGE; exit 1 ;;
    -m | --mail) MAIL=1 ;;
    -p | --passive) PASSIVE=1 ;;
    -s | --sdm) SDM=1;;
    --excl) DOEXCL=1;;
    -V) echo "${PGM}: v${VER}"; exit 3 ;;
    -x)  set -x ;;
    --mailto) MAILTO=$2; MAIL=1; shift ;;
    *)  echo "Option $1 not known."; USAGE; exit 1 ;;
  esac
    shift
done

#--------------------------------------------------------------------------#
# Create logfile and execute steps                                         #
#--------------------------------------------------------------------------#
>${LOGFILE}
>${TMPFILE}

echo "`date` ${PGM} v${VER} at ${HOSTNAME} Started at `date`"|tee -a ${LOGFILE} ${LOG}
echo "DOEXCL=${DOEXCL} SDM=${SDM}, MAIL=${MAIL}, MAILTO=${MAILTO}"|tee -a ${LOGFILE} ${LOG}

#--------------------------------------------------------------------------#
# Check snapmirror lag                                                      #
#--------------------------------------------------------------------------#
#echo "#--------------------------------------------------------------------#">> ${LOGFILE}
#echo "Check snapmirror lag:">> ${LOGFILE}
#echo "#--------------------------------------------------------------------#">> ${TMPFILE}
#echo "Check snapmirror lag:">> ${TMPFILE}

for FILER in `cat ${FILERS}|grep -v \^#|awk -F\; '{print $1}'|grep ${FILTER}`
do
  echo "`date +%Y-%m-%d_%H:%M:%S` ${PGM}: Collecting info from filer ${FILER} ..." | tee -a ${LOGFILE}
  touch ${ERR}
  touch ${TMPFILE}
  #------------------------------------------------------------------------#
  # Check for SnapVault license (for later) first. If empty, no license                #
  #------------------------------------------------------------------------#
  SVLICENSE="`${SSH} ${FILER} license||egrep 'sv_ontap|SnapVault'|awk '{print $2} '|grep -v not`"
  echo "${FILER} has SV-lic:${SVLICENSE}" >> ${LOGFILE}

  for VFILER in `${SSH} ${FILER} vfiler status|grep running|awk '{print $1}'`
  do
    echo "Collecting SnapMirror info from vfiler ${FILER}/${VFILER} ..."|tee -a ${LOGFILE}
    HEADER="* ${FILER}/${VFILER} SnapMirror lag:"
    #for LINE in `${SSH} ${FILER} vfiler run ${VFILER} snapmirror status | grep ":" | grep -v "\- " | grep -v Source | grep -v ' 0[0-9]:' | grep -v ' 1[0-9]:' | grep -v ' 2[0-9]:' | awk '{print $1"?"$2"?"$3"?"$4"?"$5}'`
    for LINE in `${SSH} ${FILER} vfiler run ${VFILER} snapmirror status | grep ":" | grep -v "\- " | grep -v ' 0[0-9]:' | grep -v ' 1[0-9]:' | grep -v ' 2[0-9]:' | awk '{print $1"?"$2"?"$3"?"$4"?"$5}'`
    do
      if  [ "${VFILER}" == "vfiler0" ]
      then 
        SRCFILER="`echo ${LINE} | awk -F "?" '{print $1}'| cut -d\: -f1|cut -d\- -f1`"
        SRCVOL="`echo ${LINE} | awk -F "?" '{print $1}'| cut -d\: -f2`"
        DESTFILER="`echo ${LINE} | awk -F "?" '{print $2}'| cut -d\: -f1|cut -d\- -f1`"
        DESTVOL="`echo ${LINE} | awk -F "?" '{print $2}'| cut -d\: -f2`"
        STATE=`echo ${LINE} | awk -F "?" '{print $3}'`
        LAG=`echo ${LINE} | awk -F "?" '{print $4}'`
        STATUS="`echo ${LINE} | awk -F "?" '{print $5}'`"
        if  [ "${STATE}" != "Source" ]
        then

        if  [ "${STATUS}" == "Pending" ]
        then
          echo "Checking sizes of ${SRCFILER}:${SRCVOL} & ${DESTFILER}:${DESTVOL} ..." >>${LOGFILE}
          SRCTOT="`${SSH} ${SRCFILER} df -k ${SRCVOL}|grep -v Filesystem|grep -v snapshot|awk '{print $2}'|sed -e s/KB//g`"
          DESTTOT="`${SSH} ${DESTFILER} df -k ${DESTVOL}|grep -v Filesystem|grep -v snapshot|awk '{print $2}'|sed -e s/KB//g`"
          #----------------------------------------------------------------#
          # Test if source volume size is greater then destination volume  #
          #----------------------------------------------------------------#
          if [ "${SRCTOT}" -gt "${DESTTOT}" ]
          then
            if [ "${HEADER}" != "" ]; then
              echo "" |tee -a ${TMPFILE}
              echo "${HEADER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
              HEADER=""
            fi
            let "SRCTOT= ${SRCTOT} / 1048576"
            let "DESTTOT= ${DESTTOT} / 1048576"
            echo "  Source ${SRCFILER}:${SRCVOL} is ${SRCTOT}GB & destination ${DESTFILER}:${DESTVOL} is ${DESTTOT}GB!" | tee -a ${TMPFILE} ${ERR} ${LOGFILE}
            if [ ${PASSIVE} ]; then
              DATI="`date +%Y-%m-%d-%H-%M-%S`"
# 1|nlnaf25: CI=nlnaf25 Host Down nlnaf25 DFM_EVENT_ID=0000000 (YYYY-MM-DD-hh-mm-ss).|MONID=MAS.NL.1.nlnaf25-SOS;CLASS=ZZ-Event.Storage.Storage;
              PASSIVESTR="1|${PGM}: CI=${FILER} SnapMirror has a high lag and is Pending. Source ${SRCFILER}:${SRCVOL} is ${SRCTOT}GB & destination ${DESTFILER}:${DESTVOL} is ${DESTTOT}GB! (${DATI}).|MONID=${MONIDHDR}.${FILER}-SOS;CLASS=${CLASS};"
              echo "  NaCl-PassiveCheck:${PASSIVESTR}"|tee -a ${LOG}
              echo ${PASSIVESTR} >> ${PASSIVECHKDIR}/${DATI}
            fi  # ${PASSIVE}
          else
            echo "  Check messages file destination filer ${DESTFILER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
          fi
        else  # [ "${STATUS}" == "Pending" ]
          #--------------------------------------------------------------------#
          # Create output                                                      #
          #--------------------------------------------------------------------#
          echo "Check SnapMirror with snapmirror.conf at ${DESTFILER}." >>${LOGFILE}
          SMCONF="`${SSH} ${FILER} rdfile /vol/vol0/etc/snapmirror.conf|grep -v "^#"|grep ${DESTVOL}`"
          if [ "${SMCONF}" != "" ]
          then
            if [ "${HEADER}" != "" ]; then
              echo "" |tee -a ${TMPFILE}
              echo "${HEADER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
              HEADER=""
            fi
            echo "  ${DESTFILER}:${DESTVOL} found in snapmirror.conf. Action is needed. "|tee -a ${TMPFILE}|tee -a ${ERR}|tee -a ${LOGFILE}
            echo "  (${SMCONF})"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
            if [ ${PASSIVE} ]; then
              DATI="`date +%Y-%m-%d-%H-%M-%S`"
              PASSIVESTR="1|${PGM}: CI=${FILER} SnapMirror has a high lag and ${DESTFILER}:${DESTVOL} found in snapmirror.conf. Action is needed. (${DATI}).|MONID=${MONIDHDR}.${FILER}-SOS;CLASS=${CLASS};"
              echo "  NaCl-PassiveCheck:${PASSIVESTR}"|tee -a ${LOG}
              echo ${PASSIVESTR} >> ${PASSIVECHKDIR}/${DATI}
            fi  # ${PASSIVE}
          else
            #echo "${DESTFILER}:${DESTVOL} NOT found in snapmirror.conf. NO action is needed. " | tee -a ${TMPFILE}
            echo "  ${DESTFILER}:${DESTVOL} NOT found in snapmirror.conf. NO action is needed. " >>${LOGFILE}
          fi
        fi  # [ "${STATUS}" == "Pending" ]
        else  # [ "${STATE}" != "Source" ]
          if [ "${HEADER}" != "" ]; then
            echo "" |tee -a ${TMPFILE}
            echo "${HEADER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
            HEADER=""
          fi
          
# Check excludes
          EXCLUDED=""
          if [ ${DOEXCL} ]; then
            # check for Source
            ANSW1="`cat ${EXCL}|awk '{print $1}'|grep \"${SRCFILER}:${SRCVOL}\"`"
            ANSW2="`cat ${EXCL}|awk '{print $2}'|grep \${DESTFILER}:${DESTVOL}\"`"
            ANSW3="`cat ${EXCL}|awk '{print $3}'|grep ${STATE}`"
            if [ "${ANSW1}" != "" ] && [ "${ANSW2}" != "" ] && [ "${ANSW3}" != "" ]; then
              EXCLUDED=1
              echo "  ${SRCFILER}:${SRCVOL}  ${DESTFILER}:${DESTVOL}  ${STATE} = excluded. "|tee -a ${LOGFILE} ${TMPFILE}
            fi  # answ's
          fi  # excl

          if [ ! ${EXCLUDED} ]; then
            echo "  ${SRCFILER}:${SRCVOL}	${DESTFILER}:${DESTVOL}	${STATE}	${LAG}	${STATUS} "|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
          fi  # EXCLUDED

        fi  # [ "${STATE}" != "Source" ]

      else  # [ "${VFILER}" == "vfiler0" ]
        #------------------------------------------------------------------#
        # Check snapmirror status of vfilers                               #
        #------------------------------------------------------------------#
        SRCFILER="`echo ${LINE} | awk -F "?" '{print $1}'| cut -d\: -f1|cut -d\- -f1`"
        SRCVOL="`echo ${LINE} | awk -F "?" '{print $1}'| cut -d\: -f2`"
        DESTFILER="`echo ${LINE} | awk -F "?" '{print $2}'| cut -d\: -f1|cut -d\- -f1`"
        DESTVOL="`echo ${LINE} | awk -F "?" '{print $2}'| cut -d\: -f2`"
        STATE=`echo ${LINE} | awk -F "?" '{print $3}'`
        LAG=`echo ${LINE} | awk -F "?" '{print $4}'`
        STATUS="`echo ${LINE} | awk -F "?" '{print $5}'`"
        if  [ "${STATUS}" == "Pending" ]
        then
          echo "Checking sizes of ${SRCFILER}:${SRCVOL} & ${DESTFILER}:${DESTVOL} ..." >>${LOGFILE}
          DESTTOT="`${SSH} ${DESTFILER} df -k ${DESTVOL} | grep -v Filesystem|grep -v snapshot|awk '{print $2}'|sed -e s/KB//g`"
          let "DESTTOT= ${DESTTOT} / 1048576"
          if [ "${HEADER}" != "" ]; then
            echo "" |tee -a ${TMPFILE}
            echo "${HEADER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
            HEADER=""
          fi
          echo "  Source volume size > then destination ${DESTFILER}:${DESTVOL}, which is ${DESTTOT}GB!"|tee -a ${TMPFILE}|tee -a ${ERR}|tee -a ${LOGFILE}
          echo "  Check messages file destination filer ${DESTFILER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
          if [ ${PASSIVE} ]; then
            DATI="`date +%Y-%m-%d-%H-%M-%S`"
            PASSIVESTR="1|${PGM}: CI=${VFILER} SnapMirror has a high lag and Source volume size > then destination ${DESTFILER}:${DESTVOL}, which is ${DESTTOT}GB! Check messages file destination filer ${DESTFILER} (${DATI}).|MONID=${MONIDHDR}.${VFILER}-VST;CLASS=${CLASS};"
            echo "  NaCl-PassiveCheck:${PASSIVESTR}"|tee -a ${LOG}
            echo ${PASSIVESTR} >> ${PASSIVECHKDIR}/${DATI}
          fi  # ${PASSIVE}
        else
          echo "Check snapmirror with snapmirror.conf at ${DESTFILER}." >>${LOGFILE}
          CUSTOMERCODE=`echo ${VFILER} | cut -c7-`
          SMCONF="`${SSH} ${FILER} rdfile /vol/vol0_vfiler${CUSTOMERCODE}/etc/snapmirror.conf|grep -v "^#" | grep ${DESTVOL}`"
          #--------------------------------------------------------------------#
          # Create output                                                      #
          #--------------------------------------------------------------------#
          if [ "${SMCONF}" != "" ]
          then
            if [ "${HEADER}" != "" ]; then
              echo "" |tee -a ${TMPFILE}
              echo "${HEADER}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
              HEADER=""
            fi
            echo "  ${SRCFILER}:${SRCVOL}	${DESTFILER}:${DESTVOL}	${STATE}	${LAG}	${STATUS} "|tee -a ${TMPFILE}|tee -a ${ERR}|tee -a ${LOGFILE}
            echo "  ${DESTFILER}:${DESTVOL} found in snapmirror.conf. Action is needed."|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
            echo "  (${SMCONF})"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
            if [ ${PASSIVE} ]; then
              DATI="`date +%Y-%m-%d-%H-%M-%S`"
              PASSIVESTR="1|${PGM}: CI=${FILER} SnapMirror has a high lag and ${DESTFILER}:${DESTVOL} found in snapmirror.conf. Action is needed. (${DATI}).|MONID=${MONIDHDR}.${FILER}-SOS;CLASS=${CLASS};"
              echo "  NaCl-PassiveCheck:${PASSIVESTR}"|tee -a ${LOG}
              echo ${PASSIVESTR} >> ${PASSIVECHKDIR}/${DATI}
            fi  # ${PASSIVE}
          else
            echo "${SRCFILER}:${SRCVOL}	${DESTFILER}:${DESTVOL}	${STATE}	${LAG}	${STATUS} "|tee -a ${LOGFILE}
            echo "${DESTFILER}:${DESTVOL} NOT found in snapmirror.conf. NO action is needed."|tee -a ${LOGFILE}
          fi  # else [ "${SMCONF}" != "" ]
        fi  # else [ "${STATUS}" == "Pending" ]
      fi  # else [ "${VFILER}" == "vfiler0" ]
    done  # snapmirror status


  #------------------------------------------------------------------------#
  # Collecting SnapVault info when SnapVault license is available          #
  #------------------------------------------------------------------------#
    if [ "${SVLICENSE}" != "" ]
    then

    echo "Collecting SnapVault info from vfiler ${FILER}/${VFILER} ..."|tee -a ${LOGFILE}
    for LINE in `${SSH} ${FILER} vfiler run ${VFILER} snapvault status | grep ':'|grep -v 'Uninitialized'|grep -v ' 0[0-9]:'|grep -v ' 1[0-9]:'|grep -v ' 2[0-4]:'|awk '{print $1"?"$2"?"$3"?"$4"?"$5}'` 
    do
      #------------------------------------------------------------------------#
      # Get destination and source qtree                                       #
      #------------------------------------------------------------------------#
      SRCQTREE=`echo ${LINE} | awk -F "?" '{print $1}'`
      DESTQTREE=`echo ${LINE} | awk -F "?" '{print $2}'`
      STATE=`echo ${LINE} | awk -F "?" '{print $3}'`
      LAG=`echo ${LINE} | awk -F "?" '{print $4}'`
      STATUS=`echo ${LINE} | awk -F "?" '{print $5}'`

# Check excludes
      EXCLUDED=""
      if [ ${DOEXCL} ]; then
        # check for Source
        ANSW1="`cat ${EXCL}|awk '{print $1}'|grep ${SRCQTREE}`"
        ANSW2="`cat ${EXCL}|awk '{print $2}'|grep ${DESTQTREE}`"
        ANSW3="`cat ${EXCL}|awk '{print $3}'|grep ${STATE}`"
        if [ "${ANSW1}" != "" ] && [ "${ANSW2}" != "" ] && [ "${ANSW3}" != "" ]; then
          EXCLUDED=1
          echo "  ${SRCQTREE}  ${DESTQTREE}  ${STATE} = excluded. "|tee -a ${LOGFILE} ${TMPFILE}
        fi  # answ's
      fi  # excl
      #------------------------------------------------------------------------#
      # Create output                                                          #
      #------------------------------------------------------------------------#
      ERROR=`${SSH} ${FILER} snapvault status -l ${DESTQTREE}|grep 'Current Transfer Error:'|cut -d: -f2`
      echo "" |tee -a ${TMPFILE}
      if [ ! ${EXCLUDED} ]; then
        echo "* ${FILER}/${VFILER} SnapVault lag:" |tee -a ${TMPFILE} ${ERR} ${LOGFILE}
        echo "  ${SRCQTREE}  ${DESTQTREE}  ${STATE}  ${LAG}  ${STATUS}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
        echo "  SnapVault Status Error:${ERROR}"|tee -a ${TMPFILE} ${ERR} ${LOGFILE}
      fi  # excluded
      if [ ${PASSIVE} ]; then
        DATI="`date +%Y-%m-%d-%H-%M-%S`"
        PASSIVESTR="1|${PGM}: CI=${FILER} SnapVault (${SRCQTREE} ${DESTQTREE}) at ${FILER}/${VFILER} has the lag:${LAG} , status:${STATUS} and error:${ERROR} (${DATI}).|MONID=${MONIDHDR}.${FILER}-SOS;CLASS=${CLASS};"
        echo "  NaCl-PassiveCheck:${PASSIVESTR}"|tee -a ${LOG}
        echo ${PASSIVESTR} >> ${PASSIVECHKDIR}/${DATI}
      fi  # ${PASSIVE}
    done  # snapvault status

    fi  # if [ "${LICENSE}" != "" ]
  done    # per vfiler

# Sending to SDM (per filer = 1 CI)
  if [ ${SDM} ]; then
# When not empty, mail
    if [ -s ${ERR} ]; then
      echo "ERR:"
      cat ${ERR}
      CREATE_SDM_TICKET ${FILER} "Query" ${ERR}
      cp /dev/null ${ERR}
##    cat ${ERR}|mailx -s "SDM:${FILER} [${PGM} v${VER}]" ${MAILTO}
    fi  # [ -s ${ERR} ]
  fi  # SDM

done  # per filer

if [ ${MAIL} ]; then
# When not empty, mail
  if [ -s ${TMPFILE} ]; then
    cat ${TMPFILE} | mailx -s "@${HOSTNAME}: Check SnapMirror-, SnapVault-lags & Old snaps [${PGM} v${VER}]" ${MAILTO}
    echo "Mailed to ${MAILTO}"|tee -a ${LOGFILE} ${LOG}
    cp ${TMPFILE} ${HOME}/log/${PGM}.tmp
  fi  # -s ${TMPFILE}
fi

rm ${TMPFILE} ${ERR}
echo "`date` ${PGM} finished."|tee -a ${LOG}
exit 0