#!/bin/sh
# File	: proc_decarbon.sh, 201019
# By	: Maarten.deBoer@Atos.net, 201019
# Subject	: SCript to process input-files with "decarbon" info
#(0.2),201019	: Mod; proc /file. Move file to proc-ed-dir
#(0.3),201102	: Restructured for NetApp
#(0.4),201102	: Added EMC & PROCDATE
#(0.5),201104	: Mod. UNKNOWN_CSV
#(0.6),201104	: Add detail for later usage at upd_
#(0.7),201109	: Add move versions. Added LOG for INFILE. Started with disks
#(0.8),201110	: Cont. with cdot-disks (using TMP.pools & TMP.disks)
#(0.9),201116	: Add USAGE
#(0.10),201118	: Add VNX
#(0.11),201119	: Mod. disks (counts) & (no)MOVE
#(0.12),201124	: Added CLEANUP (to clean (move) all files fomr the day)
#(0.13),201126	: Added POOLS & DISKS
#(0.14),201130	: Mod. DETAIL added ${OWNER}/${ARRYA}/${SYSNAME} as 3rd field after CLUSTERNAME
#(0.15),201203	: PROCDATE = today
#(0.16),201211	: Add date .CSV
#(0.17),201216	: Remove non spec chars in CSV-file & PROCDATE in _CSV-files (in stead of (runninf)date
#(0.18),210105	: Mod's. Added DPA processing
#(0.19),210105	: Added DPA_SSD_DISKS = ../etc/dpa-ssd-disks.asc
#(0.20),210108	: Mod. DPA part
#(0.21),210203	: Mod. DPA part with no Date then change with date. Added some at CLEANUP. PROCDIR -> HISTORYDIR
#(0.22),210216	: Add 7mode-disks (again)
#(0.23),210302	: Mod. DPA part. When "global" name, then used ClusterID
#(0.24),210311	: Added -;${OWNER};netapp;7mode;", --nocleanup
#(0.25),210315	: Mod. DPA_ScheduledReport filename
#(0.26),210316	: Mod. cDOT & 7-mode disks from DE
#(0.27),210317	: Add datadomain_disk-sum.csv
PGM=`basename $0|cut -d\. -f1`
VER="0.27"
TMP="/tmp/${PGM}.$$"
BASEDIR="/appl/netapp/decarbon"
LOG="${BASEDIR}/log/${PGM}.log"
INDIR="${BASEDIR}/input"
HISTORYDIR="${BASEDIR}/history"
RCFILE="${BASEDIR}/etc/${PGM}.rc"
MAILTO="maarten.deboer@atos.net"
MAIL=""
DETAILDIR="${BASEDIR}/details"
MOVE=1
CLEANUP=1
POOLS=""
DISKS=1
DPA_SSD_DISKS="${BASEDIR}/etc/dpa-ssd-disks.asc"

#PROCDATE=`date +%Y-%m-%d --date=yesterday`
PROCDATE=`date +%Y-%m-%d`

USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "Options          : "
  echo "     -h | --help : This help"
  echo "     -d | --date : change process-Date (${PROCDATE})"
  echo "     -r | --rc   : change Rcfile (${RCFILE})"
  echo "     --indir     : change INputDIRectory (${INDIR})"
  echo "     --nocleanup : do NO CLEANUP (move all files from date (${PROCDATE}))"
  echo "     --nomove    : do NO MOVE of INputDIR-data (AND NO CLEANUP)"
  echo "     -V          : Version"
}
while [ ${#} -gt 0 ]
do
  case "${1}" in
    -h | --help)  USAGE; exit 1 ;;
    -d | --date) PROCDATE="${2}"; shift ;;
    -r | --rc) RCFILE="${2}"; shift ;;
    --indir) INDIR="${2}"; shift ;;
    --nocleanup) CLEANUP="" ;;
    --nomove) MOVE=""; CLEANUP="" ;;
    -x)  set -x ;;
    -V)  echo "Version=${VER}" ; exit 1 ;;
    *) echo "Option ${1} not found." ; USAGE; exit 1 ;;
  esac
  shift
done

echo "`date` ${PGM} ${VER} started."|tee -a ${LOG}
touch ${TMP}

if [ -f ${RCFILE} ]; then
  echo "  RCFILE(${RCFILE}) found. Using it ..."|tee -a ${LOG}
  . ${RCFILE}
  sleep 1
fi
CLUSTERFILE="${BASEDIR}/etc/country2systems2cluster2vendor.csv"
POOLS_CSV="${BASEDIR}/data/country-systems-pools_${PROCDATE}.csv"
UNKNOWN_CSV="${BASEDIR}/data/unknown-systems_${PROCDATE}.csv"
DISKS_CSV="${BASEDIR}/data/country-systems-disks_${PROCDATE}.csv"
echo "BASEDIR=${BASEDIR}"
echo "CLUSTERFILE=${CLUSTERFILE}"
echo "DETAILDIR=${DETAILDIR}"
echo "DPA_SSD_DISKS=${DPA_SSD_DISKS}"
echo "DISKS=${DISKS}"
echo "DISKS_CSV=${DISKS_CSV}"
echo "INDIR=${INDIR}"
echo "POOLS=${POOLS}"
echo "POOLS_CSV=${POOLS_CSV}"
echo "HISTORYDIR=${HISTORYDIR}"
echo "PROCDATE=${PROCDATE}"
echo "UNKNOWN_CSV=${UNKNOWN_CSV}"
sleep 1

if [ ! -f ${CLUSTERFILE} ]; then
  echo "  NO CLUSTER-file (${CLUSTERFILE}) found. Exiting ..."|tee -a ${LOG}
  exit 4
fi  
if [ ! -d ${INDIR} ]; then
  echo "  NO in(put)-dir (${INDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 5
fi  
if [ ! -d ${HISTORYDIR} ]; then
  echo "  NO proc-ed-dir (${HISTORYDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 6
fi  
if [ ! -d ${DETAIL} ]; then
  echo "  NO detail-dir (${DETAILDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 7
fi  

touch ${TMP}
if [ ! -f ${UNKNOWN_CSV} ]; then
  echo "- ;OWNER;" >> ${UNKNOWN_CSV}
fi

# Find file's to be processed
# From TODAY
# Put the TB's from known / unknown in a TMP-file

# NetApp-versions
ls -1 ${INDIR}/n*_${PROCDATE}*_versions.csv 2> /dev/null| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${HISTORYDIR}/
done  # ls -1 *_versions.csv

if [ ${POOLS} ]; then
# NetApp-7mode (legacy)
  ls -1 ${INDIR}/n*_${PROCDATE}*_7mode-aggrs.csv 2> /dev/null| while read INFILE REST
  do
    echo "   ${INFILE} ..."|tee -a ${LOG}
# nlnaf25;Aggregate;total;used;avail;
    cat ${INFILE}|grep -v ^#|grep -v Aggregate|awk -F\; '{print $1,$2,$3}'|while read SYSNAME AGGRNAME SIZE_GB REST
    do
      if [ "${SYSNAME}" != "" ] && [ `grep ${SYSNAME} ${CLUSTERFILE}` ]; then
# SYSNAME = found in CLUSTERFILE
        CLUSTERNAME=`grep ${SYSNAME} ${CLUSTERFILE}|awk -F\; '{print $3}'`
        COUNTRY=`grep ${SYSNAME} ${CLUSTERFILE}|awk -F\; '{print $1}'`
        echo "    ${SYSNAME} => ${CLUSTERNAME} (${COUNTRY}) : ${SIZE_GB}GB"
        echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${AGGRNAME};legacy;${SIZE_GB};" >> ${TMP}.pools
        else
          echo "    ${SYSNAME} unknown"|tee -a ${LOG}
          echo "-;${SYSNAME};netapp;7mode;" >> ${UNKNOWN_CSV}
        fi  # grep ${OWNER}
      done  # INFILE
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi  # MOVE
  done  # ls -1 *_7mode-aggrs.csv

# NetApp-cDOT-aggr's(pools)
  ls -1 ${INDIR}/n*_${PROCDATE}*_cdot-aggrs.csv 2> /dev/null| while read INFILE REST
  do
    echo "   ${INFILE} ..."|tee -a ${LOG}
# FILE: aggregate;storage-type;aggregate-type;owner-name;size;usedsize;physical-used;
    cat ${INFILE}|grep -v ^#|grep -v aggregate|awk -F\; '{print $1,$2,$3,$4,$5}'|while read AGGRNAME STOR_TYPE AGGR_TYPE OWNER SIZE_GB REST
    do
      if [ "${OWNER}" != "" ] && [ `grep ${OWNER} ${CLUSTERFILE}` ]; then
# OWNER = found in CLUSTERFILE
        CLUSTERNAME=`grep ${OWNER} ${CLUSTERFILE}|awk -F\; '{print $3}'`
        COUNTRY=`grep ${OWNER} ${CLUSTERFILE}|awk -F\; '{print $1}'`
        echo "    ${OWNER} => ${CLUSTERNAME} (${COUNTRY})"

        if [ "${AGGR_TYPE}" = "ssd" ]; then
          echo "${COUNTRY};${CLUSTERNAME};${OWNER};${AGGRNAME};ssd;${SIZE_GB};" >> ${TMP}.pools
        else
          echo "${COUNTRY};${CLUSTERNAME};${OWNER};${AGGRNAME};${AGGR_TYPE};${SIZE_GB};" >> ${TMP}.pools
        fi  # ssd
      else
        echo "    ${OWNER} unknown"|tee -a ${LOG}
        echo "-;${OWNER};netapp;cdot;" >> ${UNKNOWN_CSV}
      fi  # grep ${OWNER}
    done  # cat INFILE (*_cdot-aggrs.csv)
# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi  # MOVE
  done  # ls -1 *_cdot-aggrs.csv


# EMC-VMAX-pools
ls -1 ${INDIR}/emc*_${PROCDATE}*_vmax-pools.csv 2> /dev/null| while read INFILE REST1
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# array ; tech ; pool_name ; owner ; pool_usable ; pool_used ; pool_physical ; data_ratio ; vp_ratio ; overall_ratio
  cat ${INFILE}|grep -v ^#|grep -v array|awk -F\; '{print $1,$2,$3,$5}'|while read ARRAY TECH POOLNAME USABLE_GB REST2
  do
    ARRAYNAME=`echo ${ARRAY} |sed "s/'//g"`
#      echo "    ${ARRAYNAME}"
    if [ "${ARRAYNAME}" != "" ] && [ `grep ${ARRAYNAME} ${CLUSTERFILE}` ]; then
      CLUSTERNAME=`grep ${ARRAYNAME} ${CLUSTERFILE}|awk -F\; '{print $3}'`
      COUNTRY=`grep ${ARRAYNAME} ${CLUSTERFILE}|awk -F\; '{print $1}'`
      echo "    ${ARRAYNAME} => ${CLUSTERNAME} (${COUNTRY})"

      SIZE_GB=`echo ${USABLE_GB}|cut -d\. -f1`

      if [ "${TECH}" = "ssd" ]; then
        echo "${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${POOLNAME};ssd;${SIZE_GB};" >> ${TMP}.pools
      else
        echo "${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${POOLNAME};${TECH};${SIZE_GB};" >> ${TMP}.pools
      fi  # ssd
    else
      echo "    ${ARRAYNAME} unknown"|tee -a ${LOG}
      echo "-;${ARRAYNAME};emc;vmax;" >> ${UNKNOWN_CSV}
    fi
  done  # INFILE
# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi  # MOVE
  done  # ls -1 *_vmax-pools.csv

fi  # POOLS


if [ ${DISKS} ]; then

# ------------------------------------------------------
# NetApp(cDOT-)disks

# From DE 1st: Rename those
# v_decarbon_netapp_cdot_disks_2021-03-16-11-37-54.csv
# netapp-rpt-de-carbon-kpi-s_dcc-new_2021-03-11-16-53-21_cdot-disks.csv
  ls -1 ${INDIR}/v_decarbon_netapp_cdot_disks_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST
  do
    if [ "${INFILE}" != "" ]; then
# Get only DateTime from filename
      INDATI=`echo ${INFILE}|awk -F\_ '{print $6}'|cut -d\. -f1|grep "2[0-9][0-9][0-9]-[01][0-9]-*"`
      mv ${INFILE} ${INDIR}/netapp-rpt-de-carbon-kpi-s_DE_${INDATI}_cdot-disks.csv
    fi  # INFILE}" != ""
  done  # -1 ${INDIR}/v*

# Process netapp cDOT disks
  ls -1 ${INDIR}/netapp*_${PROCDATE}*_cdot-disks.csv 2> /dev/null| while read INFILE REST1
  do
    echo "   ${INFILE} ..."|tee -a ${LOG}
# 1st ARRAY-loop: Check where (COUNTRY & CLUSTER) OWNER belongs to
# disk;owner;aggregate;model;type;usable-size;
#
    cat ${INFILE}|grep -v ^#|grep -v owner|awk -F\; '{print $2}'|sort -u|while read OWNER REST1
    do
      CLUSTERNAME=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
      if [ "${CLUSTERNAME}" != "" ]; then
        COUNTRY=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
        cat ${INFILE}|grep -v ^#|grep -v array|grep "${OWNER};"|cut -d\; -f4|sort -u|while read MODEL
        do
          echo "    ${OWNER}/${MODEL}"
          DISKCNT=`cat ${INFILE}|grep "${OWNER};"|grep "${MODEL}"|wc -l`
          DISKTYPE=`cat ${INFILE}|grep "${MODEL}"|cut -d\; -f5|sort -u|head -1`
          USABLE_SIZE=`cat ${INFILE}|grep "${MODEL}"|cut -d\; -f6|sort -u|tail -1`
          if [ "${DISKTYPE}" = "SSD" ]; then
            TYPE="SSD"
          else
            TYPE="HDD"
          fi  # DISKTECH
          echo "    ${OWNER}=>${CLUSTERNAME}(${COUNTRY}-${OWNER}):${MODEL}(${DISKTYPE}/${TYPE},${USABLE_SIZE})=${DISKCNT}"
          echo "${COUNTRY};${CLUSTERNAME};${OWNER};${MODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
        done  # cat ${INFILE} .. read MODEL

      else  # grep OWNER
        echo "    ${OWNER} unknown"|tee -a ${LOG}
        echo "-;${OWNER};netapp;cdot;" >> ${UNKNOWN_CSV}
      fi  # grep OWNER
    done  # cat INFILE .. OWNER
# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi  # MOVE
  done  # ls -1 *_cdot-disks.csv

# ---------------------------------------------------
# NetApp 7mode disks

# From DE 1st: Rename those
# v_decarbon_netapp_cdot_disks_2021-03-16-11-37-54.csv
# netapp-rpt-de-carbon-kpi-s_dcc-new_2021-03-11-16-53-21_cdot-disks.csv
  ls -1 ${INDIR}/v_decarbon_netapp_7mode_disks_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST
  do
    if [ "${INFILE}" != "" ]; then
# Get only DateTime from filename
      INDATI=`echo ${INFILE}|awk -F\_ '{print $6}'|cut -d\. -f1|grep "2[0-9][0-9][0-9]-[01][0-9]-*"`
      mv ${INFILE} ${INDIR}/netapp-rpt-de-carbon-kpi-s_DE_${INDATI}_7mode-disks.csv
    fi  # INFILE}" != ""
  done  # -1 ${INDIR}/v*

# Process netapp 7-mode disks
  ls -1 ${INDIR}/netapp*_${PROCDATE}*_7mode-disks.csv 2> /dev/null| while read INFILE REST1
  do
    echo "   ${INFILE} ..."|tee -a ${LOG}
    cat ${INFILE}|grep -v ^#|egrep -v 'DISK|-----|owner'|awk -F\; '{print $2}'|sort -u|while read OWNER REST1
    do
      CLUSTERNAME=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
      if [ "${CLUSTERNAME}" != "" ]; then
        COUNTRY=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
        DISKCNT=`cat ${INFILE}|grep -v ^#|egrep -v 'DISK|-----'|grep "${OWNER}"|wc -l`
        MODEL="7mode"
        DISKTYPE=""
        TYPE="HDD"
        USABLE_SIZE=""

        echo "    ${OWNER}=>${CLUSTERNAME}(${COUNTRY}-${OWNER}):${MODEL}(${DISKTYPE}/${TYPE},${USABLE_SIZE})=${DISKCNT}"
        echo "${COUNTRY};${CLUSTERNAME};${OWNER};${MODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks

      else  # CLUSTERNAME}" != ""
        echo "    ${OWNER} unknown"|tee -a ${LOG}
        echo "-;${OWNER};netapp;7mode;" >> ${UNKNOWN_CSV}
      fi  # CLUSTERNAME}" != ""
    done  # cat ${INFILE}
# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi
  done  # ls -1 ${INDIR}/n*


# ---------------------------------------------------
# EMC-VMAX-disks
  ls -1 ${INDIR}/emc*_${PROCDATE}*_vmax-disks.csv 2> /dev/null| while read INFILE REST1
  do
    echo "   ${INFILE} ..."|tee -a ${LOG}
# 1st ARRAY-loop: Check where (COUNTRY & CLUSTER) ARRAY belongs to
#array ; director:interface:tid ; technology ; vendor_id ; product_id ; serial_id ; total_disk_size_GB
#196801573;DF-1C:C:0;EFD;HITACHI;HUSMM808;2MV2205A;744
#
    cat ${INFILE}|grep -v ^#|grep -v array|awk -F\; '{print $1}'|sort -u|while read ARRAY REST2
    do
      ARRAYNAME=`echo ${ARRAY} |sed "s/'//g"`
# Define to which COUNTRY & CLUSTERNAME an ARRAYNAME belongs
      if [ "${ARRAYNAME}" != "" ]; then
# See if ARRAYNAME is known in CLUSTERFILE
        CLUSTERNAME=`grep "${ARRAYNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
# Check if ARRAYNAME is known. If not, then write to UNKNOWN_CSV
        if [ "${CLUSTERNAME}" != "" ]; then
          COUNTRY=`grep "${ARRAYNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
          cat ${INFILE}|grep -v ^#|grep -v array|grep "${ARRAYNAME};"|cut -d\; -f5|sort -u|while read PROD_ID
          do
#            echo "    ${PROD_ID}"
            DISKCNT=`cat ${INFILE}|grep "${ARRAYNAME};"|grep "${PROD_ID}"|wc -l`
            DISKTECH=`cat ${INFILE}|grep "${PROD_ID}"|cut -d\; -f3|sort -u|head -1`
            USABLE_SIZE=`cat ${INFILE}|grep "${PROD_ID}"|cut -d\; -f7|sort -u|tail -1`
            if [ "${DISKTECH}" = "EFD" ]; then
              TYPE="SSD"
            else
              TYPE="HDD"
            fi
            echo "    ${ARRAYNAME}=>${CLUSTERNAME}(${COUNTRY}-${ARRAYNAME}):${PROD_ID}(${TYPE})=${DISKCNT}"
            echo "${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${PROD_ID};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
          done  # cat ${INFILE} .. read PROD_ID
        else  # CLUSTERNAME}" != ""
          echo "    ${ARRAYNAME} unknown"|tee -a ${LOG}
          echo "-;${ARRAYNAME};emc;vmax;" >> ${UNKNOWN_CSV}
        fi  # CLUSTERNAME}" != ""
      fi  # ARRAYNAME}" != ""
    done  # cat INFILE .. ARRAY

# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi
  done  # ls -1 *_vmax-disks.csv

# ------------------------------------------------------
# EMC-VNX-disks
  ls -1 ${INDIR}/emc*_${PROCDATE}*_vnx-disks.csv 2> /dev/null| while read INFILE REST1
  do
    echo "  ${INFILE} ..."|tee -a ${LOG}
# 1st SYSNAME-loop: Check where (COUNTRY & CLUSTER) SYSNAME belongs to
# systemname;disk;model;type;capacity
# (type=tech)
# nlba0vnx09;0_0_0;ST910006CLAR1000;NL SAS;939194
    cat ${INFILE}|grep -v ^#|grep -v systemname|cut -d\; -f1|sort -u|while read SYSNAME REST2
    do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
      if [ "${SYSNAME}" != "" ]; then
        CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
        COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
# See if SYSNAME is known in CLUSTERFILE
        if [ "${CLUSTERNAME}" != "" ]; then
          cat ${INFILE}|grep -v ^#|grep -v systemname|grep "${SYSNAME};"|cut -d\; -f3|sort -u|while read DISKMODEL
          do
#            echo "    ${DISKMODEL}"
            DISKCNT=`cat ${INFILE}|grep "${SYSNAME};"|grep "${DISKMODEL}"|wc -l`
            DISKTECH=`cat ${INFILE}|grep "${DISKMODEL}"|cut -d\; -f4|sort -u|head -1`
            USABLE_SIZE=`cat ${INFILE}|grep "${DISKMODEL}"|cut -d\; -f5|sort -u|tail -1`
            ANSW=`echo "${DISKTECH}"|grep Flash`
            if [ "${ANSW}" != "" ]; then
              TYPE="SSD"
            else
              TYPE="HDD"
            fi
#            echo "      |${DISKMODEL}(${DISKTECH}/${TYPE})=${DISKCNT}"
            echo "      ${SYSNAME}=>${CLUSTERNAME}(${COUNTRY}-${SYSNAME}):${DISKMODEL}/${DISKTECH}(${TYPE})=${DISKCNT}(${USABLE_SIZE})"
            echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
#            echo "${COUNTRY};${CLUSTERNAME};${TECH};${DISKCNT};${TYPE};;" >> ${TMP}.disks
          done  # TECH
        else  # CLUSTERNAME}" != ""
          echo "    ${SYSNAME} (VNX) unknown"|tee -a ${LOG}
          echo "-;${SYSNAME};emc;vnx;" >> ${UNKNOWN_CSV}
        fi  # CLUSTERNAME}" != ""
      fi  # SYSNAME}" != ""
    done  # cat INFILE

# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi
  done  # ls -1 ${INDIR}/emc*

# -----------------------------------------
# DataDomain DPA
# DPA_ScheduledReport_2021-01-05-09-51-09_disk-config.csv
# When NO PROCDATE, rename with proc-date
# (0.25): Change FileName in DPA-ScheduledReport_DataDomain-Disk-Config-AT_2021-03-15-16-30-30.csv
#  if [ -f ${INDIR}/DPA_ScheduledReport.csv ]; then
#    mv ${INDIR}/DPA_ScheduledReport.csv ${INDIR}/DPA_ScheduledReport_${PROCDATE}_disk-config.csv
#  fi  # DPA_ScheduledReport.csv
#
  ls -1 ${INDIR}/DPA-ScheduledReport_*_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST1
  do
    echo "  ${INFILE} ..."|tee -a ${LOG}
# 1st SYSNAME-loop: Check where (COUNTRY & CLUSTER) SYSNAME belongs to
# Hostname,Device,Model,Serial Number,Firmware Revision,Size (B),DiskType,ClusterID
# arbueddm01.dcinabox.philips.com,1_1,HITACHI HUS72604CLAR3000,K3GEUZMB,N9CH,2990370979840,SAS,PPH_DCiaB
    cat ${INFILE}|grep -v ^#|grep -v Hostname|cut -d\, -f1|sort -u|while read SYSNAME REST2
    do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
      echo "  SYSNAME=${SYSNAME}"

      if [ "${SYSNAME}" != "" ]; then
# check if SYSNAME <> "globalname" like dds01..03 / ddn01 (from AT)
# If so, then look to the ClusterID part. And find the COUNTRY with this
        GLOBALNAME=`echo ${SYSNAME}|egrep "^ddn|^dds"`
        echo "      GLOBALNAME=${GLOBALNAME}"
        if [ "${GLOBALNAME}" != "" ]; then
          CLUSTERNAME=`grep "${SYSNAME}," ${INFILE}|grep -v ^#|grep -v Hostname|grep -v "Virtual_disk"|cut -d\, -f8|sort -u|head -1`
          COUNTRY=`grep "${GLOBALNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
        else
          CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
          COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
        fi  # GLOBALNAME}" != ""
        echo "      GLOBALNAME=${GLOBALNAME}=>CLUSTERNAME=${CLUSTERNAME}(COUNTRY=${COUNTRY})"

# See if SYSNAME is known in CLUSTERFILE
        if [ "${CLUSTERNAME}" != "" ]; then
          echo "    CLUSTERNAME=${CLUSTERNAME}"
# Exclude header "Hostname,....
# Also EXcluce "Virtual_disk"
          cat ${INFILE}|grep -v ^#|grep -v Hostname|grep -v "Virtual_disk"|grep "${SYSNAME},"|cut -d\, -f3|sort -u|while read DISKMODEL
          do
#            echo -n "      ${DISKMODEL}="
            DISKCNT=`cat ${INFILE}|grep "${SYSNAME},"|grep "${DISKMODEL}"|wc -l`
# Get disk-TYPE
            DISK_TYPE=`cat ${INFILE}|grep "${DISKMODEL}"|awk -F\, '{print $7}'|sort -u|head -1|sed 's/\r//g'`
            ANSW=`echo ${DISK_TYPE}|grep SSD`
            if [ "${ANSW}" != "" ]; then
              TYPE="SSD"
            else
              TYPE="HDD"
            fi
            BYTE_SIZE=`cat ${INFILE}|grep "${DISKMODEL}"|awk -F\, '{print $6}'|sort -u|head -1|sed 's/\r//g'`
# Removing CR (\r) at the end of the line
            SIZE_GIB=`expr ${BYTE_SIZE} / 1024 / 1024`
#            echo "${DISKCNT}(=${TYPE}):${BYTE_SIZE}B"

# Check on NO empty VAR's
            if [ "${COUNTRY}" != "" ] && [ "${CLUSTERNAME}" != "" ] && [ "${SYSNAME}" != "" ] && [ "${DISKMODEL}" != "" ] && [ "${DISKCNT}" != "" ] && [ "${TYPE}" != "" ]; then
              echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${SIZE_GIB};" >> ${TMP}.disks
              echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME} (COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) ; DISKCNT=${DISKCNT} (SIZE_GIB=${SIZE_GIB})" 
            else
              echo "      Some VAR(s) is(are) empty" |tee -a ${LOG}
              echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME}(COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) = DISKCNT=${DISKCNT}(SIZE_GIB=${SIZE_GIB})" |tee -a ${LOG}
            fi
# To pauze when SIZE_GIB=""
            if [ "${SIZE_GIB}" = "" ]; then
              sleep 5
            fi
          done  # DISKMODEL

        else  # CLUSTERNAME}" != ""
          echo "    ${SYSNAME} (DPA) unknown"|tee -a ${LOG}
          echo "-;${SYSNAME};emc;dpa;" >> ${UNKNOWN_CSV}
        fi  # CLUSTERNAME}" != ""

      fi  # SYSNAME}" != ""

    done  # cat INFILE
# Move file to PROC(-ed)DIR
    if [ ${MOVE} ]; then
      mv ${INFILE} ${HISTORYDIR}/
    fi
  done  # ls -1 ${INDIR}/DPA*


# DataDomains Disks (from nlxsdp01)
  ls -1 ${INDIR}/datadomain_${PROCDATE}_disks-sum.csv 2> /dev/null| while read INFILE REST1
  do
    echo "  ${INFILE} ..." |tee -a ${LOG}
    cat ${INFILE}|grep -v ^#|cut -d\; -f1|sort -u|while read SYSNAME REST2
    do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
      echo "  SYSNAME=${SYSNAME}"

      if [ "${SYSNAME}" != "" ]; then
        CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
        COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
        echo "    CLUSTERNAME=${CLUSTERNAME}"
# See if SYSNAME is known in CLUSTERFILE
        if [ "${CLUSTERNAME}" != "" ]; then
          cat ${INFILE}|grep -v ^#|grep -v systemname|grep "${SYSNAME};"|while read LINE
          do
# #DD_NAME;SSD;HDD;DRIVES;PROCDATE;REMARKS;
            echo ${LINE}
            DISKMODEL=""
            USABLE_SIZE=""
            TYPE="SSD"
            DISKCNT=`echo ${LINE}|awk -F\; '{print $2}'`
            if [ ${DISKCNT} -gt 0 ]; then
              echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME} (COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) ; DISKCNT=${DISKCNT} (USABLE_SIZE=${USABLE_SIZE})"
              echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
            fi  # DISKCNT} -gt 0
            TYPE="HDD"
            DISKCNT=`echo ${LINE}|awk -F\; '{print $3}'`
            if [ ${DISKCNT} -gt 0 ]; then
              echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME} (COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) ; DISKCNT=${DISKCNT} (USABLE_SIZE=${USABLE_SIZE})"
              echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
            fi  # DISKCNT} -gt 0

          done  # read LINE
        else  # CLUSTERNAME}" != ""
          echo "    ${SYSNAME} (DataDomain) unknown"|tee -a ${LOG}
          echo "-;${SYSNAME};emc;dd;" >> ${UNKNOWN_CSV}
        fi  # CLUSTERNAME}" != ""
      fi  # SYSNAME}" != ""
    done  # read SYSNAME
  done  # datadomain_disks
fi  # DISKS




# ========================================================
echo "  ============================================"

# 2nd part when TMP.pools is not empty
# TMP: ${COUNTRY};${CLUSTERNAME};${OWNER};${AGGRNAME};${AGGR_TYPE};${SIZE_GB};
# TMP: ${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${POOLNAME};${TECH};${SIZE_GB};
echo "  2nd part."|tee -a ${LOG}

# ------ pools / aggr's --------------------
if [ -s ${TMP}.pools ]; then
  echo "    pools"|tee -a ${LOG}
  echo "country; system(cluster / HA)name; vendor ; SSD [GiB]; HDD [GiB]; Legay(7mode) [GiB]; date (YYYY-MM-DD)" > ${POOLS_CSV}
  TOTAL_SSD=0 
  echo "TOTAL_SSD=${TOTAL_SSD}" > ${TMP}.ssd
  TOTAL_HDD=0 
  echo "TOTAL_OLD=${TOTAL_OLD}" > ${TMP}.old
# Legaycy = OLD
  TOTAL_OLD=0 
  echo "TOTAL_HDD=${TOTAL_HDD}" > ${TMP}.hdd
  cat ${TMP}.pools|sort -u|grep -v ^#|awk -F\; '{print $2}'|while read CLUSTER REST1
  do
    echo "  ${CLUSTER}"
    TOTAL_SSD=0 
    echo "TOTAL_SSD=${TOTAL_SSD}" > ${TMP}.ssd
    TOTAL_HDD=0 
    echo "TOTAL_HDD=${TOTAL_HDD}" > ${TMP}.hdd
    TOTAL_OLD=0 
    echo "TOTAL_OLD=${TOTAL_OLD}" > ${TMP}.old
    cat ${TMP}.pools|sort -u|grep -v ^#|grep ${CLUSTER}|awk -F\; '{print $2,$3,$4,$5,$6}'|while read CLUSTERNAME SYSTEMNAME POOLNAME TYPE SIZE_GB REST2
    do
      echo "${CLUSTERNAME};${SYSTEMNAME};${POOLNAME};${TYPE};${SIZE_GB}" >> ${DETAILDIR}/${CLUSTERNAME}_${PROCDATE}_pools.csv
      if [ "${TYPE}" = "ssd" ]; then
        TOTAL_SSD=`expr ${TOTAL_SSD} + ${SIZE_GB}`
#        echo "      + ${SIZE_GB}GB = ${TOTAL_SSD} SSD (${POOLNAME})"
        echo "TOTAL_SSD=${TOTAL_SSD}" > ${TMP}.ssd
      else  # ssd
        if [ "${TYPE}" = "legacy" ]; then
          TOTAL_OLD=`expr ${TOTAL_OLD} + ${SIZE_GB}`
#          echo "    + ${SIZE_GB}GB = ${TOTAL_OLD} OLD (${POOLNAME})"
          echo "TOTAL_OLD=${TOTAL_OLD}" > ${TMP}.old
        else  # legacy
          TOTAL_HDD=`expr ${TOTAL_HDD} + ${SIZE_GB}`
#          echo "    + ${SIZE_GB}GB = ${TOTAL_HDD} HDD (${POOLNAME})"
          echo "TOTAL_HDD=${TOTAL_HDD}" > ${TMP}.hdd
        fi  # legacy
      fi  # ssd
    done  # cat TMP
    COUNTRY=`grep ${CLUSTER} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
    VENDOR=`grep ${CLUSTER} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $4}'|sort -u|head -1`
    echo "${COUNTRY};${CLUSTER};${VENDOR};`cat ${TMP}.ssd|cut -d\= -f2`;`cat ${TMP}.hdd|cut -d\= -f2`;`cat ${TMP}.old|cut -d\= -f2`;${PROCDATE};"|tee -a ${POOLS_CSV}
    rm ${TMP}.ssd ${TMP}.hdd ${TMP}.old
  done  # cat TMP.pools

#  echo "# `date` processed by ${PGM} v${VER} at `hostname`" >> ${POOLS_CSV}

  if [ ${MAIL} ]; then
    date | mailx -a ${POOLS_CSV} -s "Pools/Aggrs CSV file [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  fi  # MAIL
  rm ${TMP}.pools 
fi  # -s ${TMP}.pools


# ------ disks --------------------
# Create ALWAY a DISKS_CSV
touch ${DISKS_CSV}
if [ -s ${TMP}.disks ]; then
  echo "    disks"|tee -a ${LOG}
  echo "country; clustername; vendor (emc|hds|netapp); SSD [nr disks]; HDD [nr disks]; date (YYYY-MM-DD);" > ${DISKS_CSV}
#
  cat ${TMP}.disks|grep -v ^#|awk -F\; '{print $2}'|sort -u|while read CLUSTERNAME REST1
# "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" 
# ! CLUSTERNAME NEED to be unique
  do
    if [ "${CLUSTERNAME}" != "" ]; then
      echo "  CLUSTERNAME=${CLUSTERNAME} disks ..."
      TOTAL_SSD_DISKS=0 
      echo "TOTAL_SSD_DISKS=${TOTAL_SSD_DISKS}" > ${TMP}.ssd-disks
      TOTAL_HDD_DISKS=0 
      echo "TOTAL_HDD_DISKS=${TOTAL_HDD_DISKS}" > ${TMP}.hdd-disks

#echo "${COUNTRY};${CLUSTERNAME};${TECH};${DISKCNT};${TYPE};;" >> ${TMP}.disks
      cat ${TMP}.disks|grep -v ^#|grep ${CLUSTERNAME}|sort -u| while read LINE
      do
#echo "LINE=${LINE}"
        COUNTRY=`echo ${LINE}|cut -d\; -f1`
        SYSNAME=`echo ${LINE}|cut -d\; -f3`
        DISKTECH=`echo ${LINE}|cut -d\; -f4`
        DISKCNT=`echo ${LINE}|cut -d\; -f5`
        TYPE=`echo ${LINE}|cut -d\; -f6`
        USABLE_GB=`echo ${LINE}|cut -d\; -f7`
#      echo "${COUNTRY};${CLUSTERNAME};${DISKTECH};${DISKCNT};${TYPE};${USABLE_GB};" 
        echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKTECH};${DISKCNT};${TYPE};${USABLE_GB};" >> ${DETAILDIR}/${COUNTRY}_${CLUSTERNAME}_${PROCDATE}_disks.csv
        if [ "${TYPE}" = "SSD" ]; then
          TOTAL_SSD_DISKS=`expr ${TOTAL_SSD_DISKS} + ${DISKCNT}`
          echo "      + ${DISKCNT} = ${TOTAL_SSD_DISKS} SSD"
          echo "TOTAL_SSD=${TOTAL_SSD_DISKS}" > ${TMP}.ssd-disks
        else  # ssd
          TOTAL_HDD_DISKS=`expr ${TOTAL_HDD_DISKS} + ${DISKCNT}`
          echo "    + ${DISKCNT} = ${TOTAL_HDD_DISKS} HDD"
          echo "TOTAL_HDD=${TOTAL_HDD_DISKS}" > ${TMP}.hdd-disks
        fi  # ssd
      done  # cat ${TMP}.disks
      COUNTRY=`grep ${CLUSTERNAME} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      VENDOR=`grep ${CLUSTERNAME} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $4}'|sort -u|head -1`
      if [ "${COUNTRY}" = "" ] || [ "${CLUSTERNAME}" = "" ] || [ "${VENDOR}" = "" ];then
        echo "  ERROR: COUNTRY=${COUNTRY};CLUSTERNAME=${CLUSTERNAME};VENDOR=${VENDOR};"|tee -a ${LOG}
        echo "  ERROR: COUNTRY=${COUNTRY};CLUSTERNAME=${CLUSTERNAME};VENDOR=${VENDOR};"|mailx -s "ERROR in ${PGM} v${VER} at `hostname`" ${MAILTO}
      fi

      if [ "${COUNTRY}" != "" ]; then
        echo "${COUNTRY};${CLUSTERNAME};${VENDOR};`cat ${TMP}.ssd-disks|cut -d\= -f2`;`cat ${TMP}.hdd-disks|cut -d\= -f2`;${PROCDATE};"|tee -a ${DISKS_CSV}
      fi  # COUNTRY}" != ""

      rm ${TMP}.ssd-disks ${TMP}.hdd-disks 
    fi  # CLUSTERNAME}" != ""
  done  # CAT ${TMP}.disks

#  echo "# `date` processed by ${PGM} v${VER} at `hostname`" >> ${DISKS_CSV}

  if [ ${MAIL} ]; then
    date | mailx -a ${DISKS_CSV} -s "Disks CSV file [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  fi  # MAIL
#  rm ${TMP}.disks
fi  # -s ${TMP}.disks

#-----------------------------------
# CLEANUP
if [ ${CLEANUP} ]; then
  ls -1 ${INDIR}/*_${PROCDATE}* 2> /dev/null | while read LINE
  do
    echo "  Moving ${LINE}"|tee -a ${LOG}
    mv ${LINE} ${HISTORYDIR}/
  done  # ls -1
# Remove others from INDIR
  rm ${INDIR}/*.png 2> /dev/null
  rm ${INDIR}/*.jpg 2> /dev/null
fi  # CLEANUP

echo "`date` ${PGM} v${VER} finished"|tee -a ${LOG}
exit 0