#!/bin/ksh
# File	: chk_environment.ksh
# By	: Maarten de Boer, 130517
# Subject	: Check (FSOD-)Environment
# With exclusions configured in: ${HOME}/etc/${PGM}.excludes
# For
# + Options (Filer & Vfiler)
#    - Auditing
#      auditlog.enable=on
#      cifs.audit.enable=off
#    - CIFS
#      cifs.tcp_window_size=2096560
#      cifs.max_mpx=253
# + Empty volumes (volumes without Qtrees)
# + Quota setting
# + Quota and volumes
# + Broken SnapMirrors
# + Broken SnapVaults
# + IFs van 10Gig (auto-10g) = "flowcontrol none"
#
#(0.2)	: Add; --nooptions + df at empty volume
#(0.3)	: Add; Check Quotas, --noemptyvol, -v
#(0.4)	: KSH(for counting)  & Add; --noquota
#(0.5)	: Add: Check Broken SM's & SV's, --noovercommit
#(0.6)	: Renamed in chk_env.ksh & flowcontrol 10G = none
#(0.7)	: CHKNOOVERCOMMIT -> CHKOVERCOMMIT, CHKNOQUOTA -> CHKQUOTA
#(0.8)	: Added ASUP
#(0.9)	: Added options.ASUP
#(0.10)	: Split CHECK_FILER & CHECK_VFILER
#(0.11)	: Add --all (check), --flowcontrol. Check --no(check) in --(check), ${ASC}
#(0.12)	: Add: for offline volumes
# set -x
PGM="`basename $0|cut -d\. -f1`"
VERSION="0.12" 
TMP="/tmp/${PGM}.$$"
WARN="/tmp/${PGM}.warn.$$"
LOG="$HOME/log/${PGM}.log"
MAIL=""
MAILFILE="${TMP}.mailfile"
MAILTO="maarten.deboer@atos.net"
HOSTNAME="`hostname|cut -d\. -f1`"
FILERS="${HOME}/etc/filers"
SSH="/usr/bin/ssh -n"
DATI="`date +%Y-%m-%d_%H-%M`"
TXT="${PGM}_${DATI}_${HOSTNAME}.txt"
SSH="/usr/bin/ssh -n"
FILTER="[?]*"
MAXLOGSIZE=1024   # In K's
USD=""
USDMAIL="prod-imi@gis.nl.ao-srv.com"
USDUSER="nl19471"
USDMSG="${TMP}.usd"
USDREPLYTO="maarten.deboer@atos.net"
USDTYPE="Incident"
MAILPERFILER=""
EXCLUDES="${HOME}/etc/${PGM}.excludes"
#CHKNOOPTIONS=""
CHKOPTIONS=""
#CHKNOEMPTYVOL=""
CHKEMPTYVOL=""
VERBOSE=""
CHKQUOTA=""
CHKOVERCOMMIT=""
CHKFC=""
CHKBSM=""
CHKBSV=""
ASC="/tmp/${PGM}.asc"
CHKVOLOFFLINE=""

# if not me (accourding to LOGNAME), then change some user related info
if [ "${LOGNAME}" != "nl19471" ]; then
  MAILTO="fsod@atos.net"
  USDUSER="nldsm01"
  USDREPLYTO="fsod@atos.net"
fi

# Functions
CHECK_FILER_OPTIONS()
{
  cat ${TMP}.options.${2}|grep -v \^#|while read LINE
  do
    OPTION="`echo ${LINE} | cut -d\= -f1`"
    SETTO="`echo ${LINE} | cut -d\= -f2`"
    VALUE="`${SSH} ${1} options ${OPTION} | awk '{print $2}'`"
    if [ "${SETTO}" != "${VALUE}" ]; then
      EXCLUDE="`echo "${1}:${OPTION}=${VALUE}" | egrep -f ${TMP}.excludes`"
      if [ "${EXCLUDE}" != "" ]; then
        echo "  (${1}:${OPTION}=${VALUE}. Must be:${SETTO})=Excluded & accepted." | tee -a ${TMP}
      else
        echo "${1}:${OPTION}=${VALUE}. Must be:${SETTO}"|tee -a ${TMP}
        echo "Filer options ${1}:${OPTION}=${VALUE}. Must be:${SETTO}"|tee -a ${WARN}
      fi
    fi
  done  # cat
}

CHECK_VFILERS_OPTIONS()
{
# Check options, per vfiler
  ${SSH} ${1} vfiler status|grep running|grep -v vfiler|awk '{print $1}'|while read VFILER
  do
    CIFSNOTRUN=""
    if [ "${2}" = "CIFS" ]; then
      # Check if CIFS is running, otherwise, no CIFS check
      CIFSNOTRUN="`${SSH} ${1} vfiler run -q ${VFILER} cifs domaininfo| grep 'CIFS is not running'`"
    fi  
    if [ "${2}" = "CIFS" ] && [ "${CIFSNOTRUN}" != "" ]; then
      echo "  At ${1}/${VFILER} ${ANSW} (for check options.${2})"|tee -a ${TMP}
    else
      cat ${TMP}.options.${2}|grep -v \^#|while read LINE
      do
# if =        
        ANSW="`echo ${LINE}|grep '='`"
        if [ "${ANSW}" != "" ]; then
          OPTION="`echo ${LINE} | cut -d\= -f1`"
          SETTO="`echo ${LINE} | cut -d\= -f2`"

# Getting this value is different as from normal filer
# Some options are not available ("No such option")
          VALUE="`${SSH} ${1} vfiler run ${VFILER} options ${OPTION} 2>/dev/null |tail -1|grep -v ${VFILER}|awk '{print $2}' 2>/dev/null`"
# Added: [ "${VALUE}" != "is" ] . Sometimes we get this value back from vfiler
          if [ "${VALUE}" != "is" ]; then

          if [ "${VALUE}" != "" ] && [ "${SETTO}" != "${VALUE}" ]; then
            EXCLUDE="`echo "${1}/${VFILER}:${OPTION}=${VALUE}" | egrep -f ${TMP}.excludes`"
            if [ "${EXCLUDE}" != "" ]; then
              echo "  (${1}/${VFILER}:${OPTION}=${VALUE}. Must be:${SETTO})=Excluded & accepted." | tee -a ${TMP}
            else
              echo "${1}/${VFILER}:${OPTION}=${VALUE}. Must be:${SETTO}"|tee -a ${TMP}
              echo "vFiler options ${1}/${VFILER}:${OPTION}=${VALUE}. Must be:${SETTO}"|tee -a ${WARN}
            fi
          fi
          else
            echo "${1}/${VFILER}:${OPTION}=${VALUE}. Wrong value. Need to be checked (by hand / running script again)."|tee -a ${TMP}|tee -a ${WARN}
          fi  # [ "${VALUE}" != "is" ]
        else
# if options >
          ANSW="`echo ${LINE}|grep '>'`"
          if [ "${ANSW}" != "" ]; then
            OPTION="`echo ${LINE} | cut -d\> -f1`"
            SETTO="`echo ${LINE} | cut -d\> -f2`"
            VALUE="`${SSH} ${1} vfiler run ${VFILER} options ${OPTION} 2>/dev/null |tail -1|grep -v ${VFILER}|awk '{print $2}' 2>/dev/null`"
            if [ ${VALUE} -lt ${SETTO} ]; then
              echo "${1}/${VFILER}:${OPTION}=${VALUE}. Must be >${SETTO}"|tee -a ${TMP}
              echo "vFiler options ${1}/${VFILER}:${OPTION}=${VALUE}. Must be >${SETTO}"|tee -a ${WARN}

            fi  # V > S

          fi  # grep '>'


        fi  # grep '='
      done  # cat
    fi  # if CIFS running ?
  done  # SSH vfiler status
}

CREATE_USD_TICKET()
# $1=Filer(CI), $2=USDTYPE, $3=USD-text
{
  echo "@REQUESTTYPE: ${2}" > ${USDMSG}
  echo "@REQUESTAREA: NL.Storage.StorageOnDemand" >> ${USDMSG}
  echo "@SEVERITY: 4" >> ${USDMSG}
  echo "@CI: ${1}" >> ${USDMSG}
  echo "@ORGANISATION: AtosOrigin.Netherlands" >> ${USDMSG}
  echo "@GROUP: NL.Storage.FSOD" >> ${USDMSG}
  echo "@REPLYTO: ${USDREPLYTO}" >> ${USDMSG}
  echo "@PROXYUSER: ${USDUSER}"  >> ${USDMSG}
  cat ${3} >> ${USDMSG}
  echo "[${PGM} v${VERSION}]" >> ${USDMSG}
  echo "@ENDOFUSDMESSAGE" >> ${USDMSG}

  cat ${USDMSG} | mailx -s "${USDUSER}: WARNING; Environment check ${1}." ${USDMAIL}

  echo "`date` ${PGM}: Mailed to USD-IMI (${USDMAIL})"|tee -a ${LOG}
}

USAGE()
{
  echo "Usage: ${PGM} <options>"
  echo "  Version: ${VERSION}"
  echo "  options          :"
  echo "    --all          : do ALL checks"
#  echo "    --nooptions    : Do NOT check the options"
  echo "    --options      : do check the Options"
#  echo "    --noemptyvol   : Do NOT check empty volumes"
  echo "    --emptyvol     : do check empty volumes"
  echo "    --quota        : do check quota at qtrees"
  echo "    --flowcontrol  : do check flowcontrol at 10Gig ports = none"
  echo "    --overcommit   : do check for overcommiting"
  echo "    --bsm          : do check for Broken SnapMirrors"
  echo "    --bsv          : do check for Broken SnapVaults"
  echo "    --voloffline   : do check for offline volumes"
  echo "    -m|--mail      : do send Mail"
  echo "    -u|--usd       : do send to USD"
  echo "    -e|--etc       : Etc/filers-file (${FILERS})"
  echo "    -f             : Filter filername (${FILTER})"
  echo "    -h|--help      : this Help"
  echo "    -v             : Verbose"
  echo "    -V             : show Version"
  echo "    -x             : set -x"
  echo "    --mailto       : change MAILTO address & do send mail (${MAILTO})"
  echo "    --mpf          : MailPerFiler (normaly all filers in 1 mail)"
}

## MAIN
# Check options
if [ $# -eq 0 ]; then
  echo "No option(s) given. So not to know what to do. Exiting..."; echo; USAGE; exit 1
fi
while [ $# -gt 0 ]
  do
  case $1 in
    -f) FILTER=$2; shift ;;
    --all) CHKBSM=1;CHKBSV=1;CHKOPTIONS=1;CHKEMPTYVOL=1;CHKQUOTA=1;CHKOVERCOMMIT=1;CHKVOLOFFLINE=1 ;;
#    --nooptions) CHKNOOPTION=1 ;;
    --options) CHKOPTIONS=1 ;;
#    --noemptyvol) CHKNOEMPTYVOL=1 ;;
    --emptyvol) CHKEMPTYVOL=1 ;;
    --quota) CHKQUOTA=1 ;;
    --flowcontrol) CHKFC=1 ;;
    --overcommit) CHKOVERCOMMIT=1 ;;
    --bsm) CHKBSM=1 ;;
    --bsv) CHKBSV=1 ;;
    --voloffline) CHKVOLOFFLINE=1 ;;
    -m | --mail) MAIL=1 ;;
    -e | --etc) FILERS=$2; shift ;;
    --mailto) MAILTO=$2; MAIL=1; shift ;;
    --mpf) MAILPERFILER=1;;
    -u | --usd) USD=1;;
    -h | --help) USAGE; exit 1 ;;
    -v) VERBOSE=1 ;;
    -V) echo "${PGM}: v${VERSION}"; exit 3 ;;
    -x)  set -x ;;
    *)  echo "Option ${1} not known. Exiting..."; echo; USAGE; exit 1 ;;
  esac
    shift
done

echo "`date` ${PGM} (v${VERSION}) started (LOGNAME=${LOGNAME})."|tee -a ${LOG}
echo "CHKBSM=${CHKBSM}"
echo "CHKBSV=${CHKBSV}"
echo "CHKEMPTYVOL=${CHKEMPTYVOL}"
echo "CHKOPTIONS=${CHKOPTIONS}"
echo "CHKOVERCOMMIT=${CHKOVERCOMMIT}"
echo "CHKQUOTA=${CHKQUOTA}"
echo "CHKVOLOFFLINE=${CHKVOLOFFLINE}"
echo "ETC=${ETC}"
echo "EXCLUDES=${EXCLUDES}"
echo "FILTER=${FILTER}"
echo "MAIL=${MAIL}"
echo "MAILTO=${MAILTO}"
echo "MAILPERFILER=${MAILPERFILER}"
echo "USD=${USD}"
echo "VERBOSE=${VERBOSE}"

touch ${TMP} ${TMP}.excludes ${USDMSG} ${WARN} ${CIFSWARN}
# Check & move LOG-file if longer then max.
LOGSIZE=`du -ka ${LOG} | cut -f1`
if [ ${LOGSIZE} -ge ${MAXLOGSIZE} ]; then
  mv ${LOG} ${LOG}.old
  touch ${LOG}
fi

if [ ${CHKOPTIONS} ]; then
  # Init options-file
  cat << !EOF >> ${TMP}.options.Audit
# Audits
auditlog.enable=on
cifs.audit.enable=off
!EOF
cat << !EOF >> ${TMP}.options.CIFS
cifs.tcp_window_size>2000000
cifs.max_mpx=253
!EOF
cat << !EOF >> ${TMP}.options.ASUP
autosupport.enable=on
autosupport.mailhost=mailhost
autosupport.noteto=fsod@atos.net
autosupport.partner.to=support@e-storage.nl,fsod@atos.net,mdrglob@msssyscol01.bcklan.ao-srv.com
autosupport.performance_data.enable=on
autosupport.support.transport=smtp
autosupport.to=fsod@atos.net,support@e-storage.nl
!EOF
fi  # ${CHKOPTIONS}

touch ${TMP}.excludes
if [ ! -f ${EXCLUDES} ]; then
  echo "`date` No ${EXCLUDES} found. So NO excludes will be made."|tee -a ${LOG}
else
# Remove # from .excludes-file
  echo "`date` ${EXCLUDES} found. Will be used."|tee -a ${LOG}
  cat ${EXCLUDES} |grep -v ^# >> ${TMP}.excludes
fi  # {EXCLUDES}

# Start Checking (loop)
for FILER in `cat ${FILERS}|grep -v \^#|grep "${FILTER}"|awk -F\; '{print $1}'|sort`
do
  echo ""|tee -a ${TMP}
  echo "* ${FILER}"|tee -a ${TMP}
  echo "`date` ${PGM}: ${FILER}."|tee -a ${LOG}

  if [ ${CHKOPTIONS} ]; then
    echo "+ Options"|tee -a ${TMP}
    echo "- Auditing ${FILER}:"|tee -a ${TMP}
    CHECK_VFILERS_OPTIONS ${FILER} "Audit"

    echo "- CIFS ${FILER}:"|tee -a ${TMP}
    CHECK_VFILERS_OPTIONS ${FILER} "CIFS"

    echo "- AutoSUPport ${FILER}:"|tee -a ${TMP}
    CHECK_FILER_OPTIONS ${FILER} "ASUP"
  fi   # ${CHKOPTIONS}

  if [ ${CHKEMPTYVOL} ]; then
    echo "+ Empty volumes (volumes without Qtrees) ${FILER}:"|tee -a ${TMP}
    ${SSH} ${FILER} vfiler status|grep running|grep -v vfiler|awk '{print $1}'|while read VFILER
    do
      ${SSH} ${FILER} vfiler run -q ${VFILER} vol status|grep online|grep -v vol0_vfiler|awk '{print $1}'|while read VOL
      do
        QTREE="`${SSH} ${FILER} vfiler run -q ${VFILER} qtree status ${VOL}|egrep -v 'Volume|------'|awk '{print $2}'|tail -1|grep -v unix`"
        if [ "${QTREE}" = "" ]; then
          echo -n "No Qtree at volume: ${FILER}/${VFILER}:${VOL} "|tee -a ${TMP}|tee -a ${WARN}
          TOTALGB="`${SSH} ${FILER} df -g ${VOL}| grep -v Filesystem|head -1|awk '{print $2}'`"
          SNAPUSEDGB="`${SSH} ${FILER} df -g ${VOL}| grep -v Filesystem|grep snapshot|awk '{print $3}'`"
          echo "${TOTALGB} total (snapshot = ${SNAPUSEDGB} used)"|tee -a ${TMP} ${WARN}
        fi  # QTREE

      done  # VOL
    done  # vfiler status
  fi  # ${CHKEMPTYVOL}

  if [ ${CHKQUOTA} ]; then
    echo "+ Quota setting ${FILER}:"|tee -a ${TMP}
    ${SSH} ${FILER} vfiler status|grep running|grep -v vfiler|sort|awk '{print $1}'|while read VFILER
    do
      if [ ${VERBOSE} ]; then
        echo "${FILER}/${VFILER}"
      fi
      ${SSH} ${FILER} vfiler run -q ${VFILER} vol status|grep online|sort|grep -v vol0_vfiler|awk '{print $1}'|while read VOL
      do
        QUOTASTATUS="`${SSH} ${FILER} vfiler run -q ${VFILER} quota status ${VOL}|cut -d\: -f2`"
        if [ "${QUOTASTATUS}" = " quotas are on." ]; then
          QTREE="`${SSH} ${FILER} vfiler run -q ${VFILER} qtree status ${VOL}|egrep -v 'Volume|------'|awk '{print $2}'|tail -1|grep -v unix`"
          QUOTA="`${SSH} ${FILER} vfiler run -q ${VFILER} quota report /vol/${VOL}/${QTREE}|grep tree|egrep -v 'Files|Type|-----'|awk '{print $6}'`"
          if [ ${VERBOSE} ]; then
            echo "${FILER}/${VFILER}:/vol/${VOL} ${QTREE} |${QUOTA}|"
          fi
          if [ "${QUOTA}" = "" ]; then
            VFILERETC="`${SSH} ${FILER} vfiler status -a ${VFILER}|grep etc|awk '{print $2}'`"
            echo "No Quota config found in etc/quotas for ${FILER}/${VFILER}:/vol/${VOL}/${QTREE}. See:"|tee -a ${TMP} ${WARN}
            ${SSH} ${FILER} rdfile ${VFILERETC}/etc/quotas|grep ${VOL}|tee -a ${TMP}

          fi  # QUOTA
          if [ "${QUOTA}" = "-" ]; then
            QTREEUSED="`${SSH} ${FILER} vfiler run -q ${VFILER} quota report /vol/${VOL}/${QTREE}|grep tree|egrep -v 'Files|Type|-----'|awk '{print $5}'`"
            echo "No Quota set on Qtree ${FILER}/${VFILER}:/vol/${VOL}/${QTREE} = ${QTREEUSED} kbytes used"|tee -a ${TMP}
          fi  # QUOTA

        else  # QUOTASTATUS
        # ao_ & av_ volumes do NOT have Quota on. So will be excluded.
          ANSW="`echo ${VOL}|egrep -v '^ao_|^av_'`"
          if [ "${ANSW}" != "" ]; then
            echo "No Quota on volume ${FILER}/${VFILER}:/vol/${VOL}"|tee -a ${TMP} ${WARN}
          fi  # ${ANSW}
        fi  # QUOTASTATUS
      done  # vol status
    done  #  Quota setting
  fi  # CHKQUOTA

  if [ ${CHKOVERCOMMIT} ]; then
    echo "+ Quota and volumes ${FILER} (overcommitting):"|tee -a ${TMP}
    ${SSH} ${FILER} vfiler status|grep running|grep -v vfiler|sort|awk '{print $1}'|while read VFILER
    do
      ${SSH} ${FILER} vfiler run -q ${VFILER} vol status|grep online|sort|grep -v vol0_vfiler|awk '{print $1}'|while read VOL
      do
        TOTQUOTA=0
        QUOTASTATUS="`${SSH} ${FILER} vfiler run -q ${VFILER} quota status ${VOL}|cut -d\: -f2`"
        if [ "${QUOTASTATUS}" = " quotas are on." ]; then
          ${SSH} ${FILER} vfiler run -q ${VFILER} quota report |grep ${VOL}|grep tree|grep -v '*'|awk '{print $6}'| while read QUOTA
          do
            if [ "${QUOTA}" = "-" ]; then
              QUOTA=999999999999999
            fi
            let "TOTQUOTA= $TOTQUOTA + $QUOTA"

          done  # QUOTA
          VOLSAVED="`${SSH} ${FILER} df -s ${VOL}|egrep -v 'Filesystem'|awk '{print $3}'`"
          VOLTOTAL="`${SSH} ${FILER} df -k ${VOL}|egrep -v 'total|snapshot'|awk '{print $2}'|sed 's/KB//g'`"
          let "QUOTASAVED= $TOTQUOTA - $VOLSAVED"
          if [ ${QUOTASAVED} -gt ${VOLTOTAL} ]; then
            echo "Quota to large for ${FILER}/${VFILER}:/vol/${VOL} Quota-Saved=${QUOTASAVED} > VolTot=${VOLTOTAL}"|tee -a ${TMP} ${WARN}
          fi  # ${TOTQUOTA} -gt ${VOLTOTAL} 

          if [ ${VERBOSE} ]; then
            echo "${FILER}/${VFILER}:/vol/${VOL} TotQ=${TOTQUOTA} VolT=${VOLTOTAL}"
          fi

        fi   # QUOTASTATUS
      done  # VOL
    done  # VFILER
  fi  # CHKOVERCOMMIT

  if [ ${CHKBSM} ]; then
    echo "+ Broken SnapMirrors ${FILER}:"|tee -a ${TMP}
    ${SSH} ${FILER} snapmirror status | grep Broken | tee -a ${TMP}
  fi  #  ${CHKBSM}

  if [ ${CHKBSV} ]; then
    echo "+ Broken SnapVaults ${FILER}:"|tee -a ${TMP}
    ${SSH} ${FILER} snapvault status | grep Broken | tee -a ${TMP}
  fi  # ${CHKBSV} 

  if [ ${CHKFC} ]; then
    echo "+ Flowcontrol 10Gig IFs = none ${FILER}?:"|tee -a ${TMP}
    ${SSH} ${FILER} ifconfig -a| grep flags| grep ^e|grep -v PRIVATE|cut -d\: -f1|sort| while read IF
    do
      FC="`${SSH} ${FILER} ifconfig ${IF}|grep auto-10g|grep flowcontrol|awk '{print $5}'`"
      if [ "${FC}" != "" ] && [ "${FC}" != "none" ]; then
        echo "Flowcontrol at 10Gig ${IF} = ${FC} (should be none)."|tee -a ${TMP} ${WARN}
      fi  # !"none"
    done  # IF
  fi  # CHKFC

  if [ ${CHKVOLOFFLINE} ]; then
    echo "+ Offline volumes ${FILER}:"|tee -a ${TMP}
    ${SSH} ${FILER} vol status | grep offline|tee -a ${TMP}
  fi  # CHKVOLOFFLINE


# When -u | --usd, send to USD PER Filer (Filer=CI)
  if [ ${USD} ]; then
    if [ -s ${WARN} ]; then
      CREATE_USD_TICKET ${FILER} "Incident" ${WARN}
      cp /dev/null ${WARN}
    fi  # [ -s ${WARN} ]

    if [ ${MAILPERFILER} ]; then
      cat ${WARN}|unix2dos|uuencode ${TXT}|mailx -s ":${HOSTNAME}: ${FILER} Check environment at `date +%Y-%m-%d_%H:%M:%S` [${PGM} v${VERSION}]" ${MAILTO}
      echo "`date` MailedPerFiler to ${MAILTO} "|tee -a ${LOG}
    fi  # [ ${MAILPERFILER} ]
    cp /dev/null ${WARN}
  fi  # [ ${USD} ]

done  # for FILER

echo ""|tee -a ${TMP}
echo "#"|tee -a ${TMP}
echo "# Output (${TXT}) from ${HOSTNAME} at `date +%Y-%m-%d_%H:%M:%S` of ${PGM} version ${VERSION}"|tee -a ${TMP}
echo "# Options CHKBSM=${CHKBSM} CHKBSV=${CHKBSV} CHKEMPTYVOL=${CHKEMPTYVOL} CHKOPTIONS=${CHKOPTIONS} CHKOVERCOMMIT=${CHKOVERCOMMIT} CHKQUOTA=${CHKQUOTA} CHKVOLOFFLINE=${CHKVOLOFFLINE} "|tee -a ${TMP}
echo "# (etc)FILERS=${FILERS}, FILTER=${FILTER}, sentMAIL=${MAIL}, MAILTO=${MAILTO} MPF=${MAILPERFILER} "|tee -a ${TMP}
echo "# Ready at `date`"|tee -a ${TMP}

# save the "output"(tmp) file to .out
cp ${TMP} /tmp/${PGM}.out

# Mail the info
if [ ${MAIL} ]; then
  cp ${TMP} ${ASC}
  date|mailx -a ${ASC} -s ":${HOSTNAME}: Check filer environment (.ASC) at `date +%Y-%m-%d_%H:%M:%S` [${PGM} v${VERSION}]" ${MAILTO}
  echo "`date` ${PGM}: Mailed (.ASC) to ${MAILTO}."|tee -a ${LOG}
  rm ${ASC}
fi  # if [ ${MAIL} ]

cp ${TMP} /tmp/${PGM}.txt

# Cleanup
rm ${TMP} ${TMP}.options.* ${TMP}.excludes ${USDMSG} ${WARN} 
echo "`date` ${PGM} (v$VERSION) finished."|tee -a $LOG
exit 0