#!/bin/sh
# File	: proc_decarbon-disks.sh, 201019
# By	: Maarten.deBoer@Atos.net, 201019
# Subject	: SCript to process input-files with "decarbon" info
#(0.2),201019	: Mod; proc /file. Move file to proc-ed-dir
#(0.3),201102	: Restructured for NetApp
#(0.4),201102	: Added EMC & PROCDATE
#(0.5),201104	: Mod. UNKNOWN_CSV
#(0.6),201104	: Add detail for later usage at upd_
#(0.7),201109	: Add move versions. Added LOG for INFILE. Started with disks
#(0.8),201110	: Cont. with cdot-disks (using TMP.pools & TMP.disks)
#(0.9),201116	: Add USAGE
#(0.10),201118	: Add VNX
#(0.11),201119	: Mod. disks (counts) & (no)MOVE
#(0.12),201124	: Added CLEANUP (to clean (move) all files fomr the day)
#(0.13),201126	: Added POOLS & DISKS
#(0.14),201130	: Mod. DETAIL added ${OWNER}/${ARRYA}/${SYSNAME} as 3rd field after CLUSTERNAME
#(0.15),201203	: PROCDATE = today
#(0.16),201211	: Add date .CSV
#(0.17),201216	: Remove non spec chars in CSV-file & PROCDATE in _CSV-files (in stead of (runninf)date
#(0.18),210105	: Mod's. Added DPA processing
#(0.19),210105	: Added DPA_SSD_DISKS = ../etc/dpa-ssd-disks.asc
#(0.20),210108	: Mod. DPA part
#(0.21),210203	: Mod. DPA part with no Date then change with date. Added some at CLEANUP. PROCDIR -> HISTORYDIR
#(0.22),210216	: Add 7mode-disks (again)
#(0.23),210302	: Mod. DPA part. When "global" name, then used ClusterID
#(0.24),210311	: Added -;${OWNER};netapp;7mode;", --nocleanup
#(0.25),210315	: Mod. DPA_ScheduledReport filename
#(0.26),210316	: Mod. cDOT & 7-mode disks from DE
#(0.27),210317	: Add datadomain_disk-sum.csv
#(0.28),210318	: Renamed (from proc_decarbon.sh), Removes all POOL related, Add. Unity (EMC), At DPA-BYTE_SIZE: sed 's/,.//g'`
#(0.29),210322	: Add --history to copy input-files from history of the DATE
#(0.30),210326	: Add --rem-unknown for remoding data/unknown files 1st
#(0.31),210329	: Add 7mode DFM files
#(0.32),210408	: Add; ANSW=`echo "${DISKTYPE}"|grep -i ssd`, RM_DETAIL
PGM=`basename $0|cut -d\. -f1`
VER="0.32"
TMP="/tmp/${PGM}.$$"
BASEDIR="/appl/netapp/decarbon"
LOG="${BASEDIR}/log/${PGM}.log"
INDIR="${BASEDIR}/input"
HISTORYDIR="${BASEDIR}/history"
RCFILE="${BASEDIR}/etc/${PGM}.rc"
MAILTO="maarten.deboer@atos.net"
MAIL=""
DETAILDIR="${BASEDIR}/details"
MOVE=1
CLEANUP=1
DISKS=1
DPA_SSD_DISKS="${BASEDIR}/etc/dpa-ssd-disks.asc"
PROC_HISTORY=""
RM_UNKNOWN=""
RM_DETAIL=""
TMPDIR="/tmp/${PGM}"

#PROCDATE=`date +%Y-%m-%d --date=yesterday`
PROCDATE=`date +%Y-%m-%d`

USAGE()
{
  echo "Usage: ${PGM} [options]"
  echo "Options          : "
  echo "     -h | --help : This help"
  echo "     -d | --date : change process-Date (${PROCDATE})"
  echo "     --history   : copy HISTORY files 1st into INPUTdir"
  echo "     -r | --rc   : change Rcfile (${RCFILE})"
  echo "     --indir     : change INputDIRectory (${INDIR})"
  echo "     --nocleanup : do NO CLEANUP (move all files from date (${PROCDATE}))"
  echo "     --nomove    : do NO MOVE of INputDIR-data (AND NO CLEANUP)"
  echo "     --rm-unknown: Remove unknown-system (data) 1st"
  echo "     --rm-detail : Remove Detail (data) 1st"
  echo "     -V          : Version"
}
while [ ${#} -gt 0 ]
do
  case "${1}" in
    -h | --help)  USAGE; exit 1 ;;
    -d | --date) PROCDATE="${2}"; shift ;;
    -r | --rc) RCFILE="${2}"; shift ;;
    --indir) INDIR="${2}"; shift ;;
    --nocleanup) CLEANUP="" ;;
    --nomove) MOVE=""; CLEANUP="" ;;
    --history) PROC_HISTORY=1 ;;
    --rm-unknown) RM_UNKNOWN=1 ;;
    --rm-detail) RM_DETAIL=1 ;;
    -x)  set -x ;;
    -V)  echo "Version=${VER}" ; exit 1 ;;
    *) echo "Option ${1} not found." ; USAGE; exit 1 ;;
  esac
  shift
done

echo "`date` ${PGM} ${VER} started."|tee -a ${LOG}
touch ${TMP}

if [ -f ${RCFILE} ]; then
  echo "  RCFILE(${RCFILE}) found. Using it ..."|tee -a ${LOG}
  . ${RCFILE}
  sleep 1
fi
CLUSTERFILE="${BASEDIR}/etc/country2systems2cluster2vendor.csv"
UNKNOWN_CSV="${BASEDIR}/data/unknown-systems_${PROCDATE}.csv"
DISKS_CSV="${BASEDIR}/data/country-systems-disks_${PROCDATE}.csv"
echo "BASEDIR=${BASEDIR}"
echo "CLUSTERFILE=${CLUSTERFILE}"
echo "DETAILDIR=${DETAILDIR}"
echo "DPA_SSD_DISKS=${DPA_SSD_DISKS}"
echo "DISKS=${DISKS}"
echo "DISKS_CSV=${DISKS_CSV}"
echo "INDIR=${INDIR}"
echo "HISTORYDIR=${HISTORYDIR}"
echo "PROCDATE=${PROCDATE}"
echo "PROC_HISTORY=${PROC_HISTORY}"
echo "UNKNOWN_CSV=${UNKNOWN_CSV}"
echo "RM_UNKNOWN=${RM_UNKNOWN}"
echo "RM_DETAIL=${RM_DETAIL}"
sleep 1

if [ ! -f ${CLUSTERFILE} ]; then
  echo "  NO CLUSTER-file (${CLUSTERFILE}) found. Exiting ..."|tee -a ${LOG}
  exit 4
fi  
if [ ! -d ${INDIR} ]; then
  echo "  NO in(put)-dir (${INDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 5
fi  
if [ ! -d ${HISTORYDIR} ]; then
  echo "  NO proc-ed-dir (${HISTORYDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 6
fi  
if [ ! -d ${DETAILDIR} ]; then
  echo "  NO detail-dir (${DETAILDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 7
fi  

touch ${TMP}

# When RM_UNKNOWN, then remove data/unknown-systems-file (of the date) 1st
if [ ${RM_UNKNOWN} ]; then
  if [ -f ${UNKNOWN_CSV} ]; then
    echo "    Removing UNKNOWN_CSV (${UNKNOWN_CSV})"|tee --a ${LOG}
    rm ${UNKNOWN_CSV}
  fi  # -f UNKNOWN_CSV
fi  # RM_UNKNOWN


if [ ! -f ${UNKNOWN_CSV} ]; then
  echo "- ;OWNER;" >> ${UNKNOWN_CSV}
fi

# When PROC_HISTORY, then copy HISTORY files of the DATE 1st
if [ ${PROC_HISTORY} ]; then
  echo "    Copying files of ${PROCDATE} from ${HISTORYDIR} into ${PROCDATE} ..."|tee -a ${LOG}
  cp ${HISTORYDIR}/*${PROCDATE}* ${INDIR}
fi

# When --rm-detail
if [ ${RM_DETAIL} ]; then
  echo "    Removing Detail information (in ${DETAILDIR}) 1st."|tee -a ${LOG}
  rm ${DETAILDIR}/*${PROCDATE}*.csv
fi


# Find file's to be processed
# From TODAY
# Put the TB's from known / unknown in a TMP-file
# ------------------------------------------------------
# NetApp(cDOT-)disks
#
# From DE 1st: Rename those
# v_decarbon_netapp_cdot_disks_2021-03-16-11-37-54.csv
# netapp-rpt-de-carbon-kpi-s_dcc-new_2021-03-11-16-53-21_cdot-disks.csv
  ls -1 ${INDIR}/v_decarbon_netapp_cdot_disks_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST
  do
    if [ "${INFILE}" != "" ]; then
# Get only DateTime from filename
      INDATI=`echo ${INFILE}|awk -F\_ '{print $6}'|cut -d\. -f1|grep "2[0-9][0-9][0-9]-[01][0-9]-*"`
      mv ${INFILE} ${INDIR}/netapp-rpt-de-carbon-kpi-s_DE_${INDATI}_cdot-disks.csv
    fi  # INFILE}" != ""
  done  # -1 ${INDIR}/v*

# Process netapp cDOT disks
ls -1 ${INDIR}/netapp*_${PROCDATE}*_cdot-disks.csv 2> /dev/null| while read INFILE REST1
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# 1st ARRAY-loop: Check where (COUNTRY & CLUSTER) OWNER belongs to
# disk;owner;aggregate;model;type;usable-size;
#
  cat ${INFILE}|grep -v ^#|grep -v owner|awk -F\; '{print $2}'|sort -u|while read OWNER REST1
  do
    CLUSTERNAME=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
    if [ "${CLUSTERNAME}" != "" ]; then
      COUNTRY=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      cat ${INFILE}|grep -v ^#|grep -v array|grep "${OWNER};"|cut -d\; -f4|sort -u|while read MODEL
      do
        DISKCNT=`cat ${INFILE}|grep "${OWNER};"|grep "${MODEL}"|wc -l`
        DISKTYPE=`cat ${INFILE}|grep "${MODEL}"|cut -d\; -f5|sort -u|head -1`
        USABLE_SIZE=`cat ${INFILE}|grep "${MODEL}"|cut -d\; -f6|sort -u|tail -1`
        echo "    ${OWNER}/${MODEL} => DISKTYPE=${DISKTYPE}"
        ANSW=`echo "${DISKTYPE}"|grep -i ssd`
        if [ "${ANSW}" != "" ]; then
          TYPE="SSD (${DISKTYPE})"
        else
          TYPE="HDD (${DISKTYPE})"
        fi
        echo "    ${OWNER}=>${CLUSTERNAME}(${COUNTRY}-${OWNER}):${MODEL}(${DISKTYPE}/${TYPE},${USABLE_SIZE})=${DISKCNT}"
        echo "${COUNTRY};${CLUSTERNAME};${OWNER};${MODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
      done  # cat ${INFILE} .. read MODEL

    else  # grep OWNER
      echo "    ${OWNER} unknown"|tee -a ${LOG}
      echo "-;${OWNER};netapp;cdot;" >> ${UNKNOWN_CSV}
    fi  # grep OWNER
  done  # cat INFILE .. OWNER
# Move file to PROC(-ed)DIR
  if [ ${MOVE} ]; then
    mv ${INFILE} ${HISTORYDIR}/
  fi  # MOVE
done  # ls -1 *_cdot-disks.csv

# ---------------------------------------------------
# PROC 7mode DFM files 1st
# Layout like: 44645_31971_2021_3_22_13_23_53.zip
#
# Define other DATE YYYY_m_dd in stead of YYYY-MM-DD
PROCDATE2=`date --date="${PROCDATE}" +%Y_%_m_%e|sed 's/ //g'`
echo "  PROCDATE2=${PROCDATE2}"
ls -1 ${INDIR}/*_${PROCDATE2}_*.zip 2> /dev/null| while read ZIPINFILE REST
do
  echo "    ${ZIPINFILE}"
  if [ ! -d ${TMPDIR} ]; then
    mkdir -p ${TMPDIR}
  fi
  unzip -f ${ZIPINFILE} -d ${TMPDIR}
  ls -1 ${TMPDIR}/*_${PROCDATE2}_*.csv | while read CSVINFILE REST2
  do
    echo "      ${CSVINFILE}"
# "Controller","Disk Name","Aggregate","Firmware Revision","Disk Vendor","Disk Model","Disk Type","Disk Shelf","Disk Bay","Disk Plex Id","Disk Size (MB)","Owner Node Id"
    cat ${CSVINFILE} | egrep -v "Controller|Totals|========"|while read LINE
    do
      CONTROLER=`echo ${LINE}|awk -F\, '{print $1}'|cut -d\. -f1`
      DISK_NAME=`echo ${LINE}|awk -F\, '{print $2}'|sed 's/disk//g'|sed 's/data//g'|sed 's/spare//g'|sed 's/parity//g'|sed 's/dparity//g'|sed 's/ //g'`
      DISK_VENDOR=`echo ${LINE}|awk -F\, '{print $5}'`
      DISK_MODEL=`echo ${LINE}|awk -F\, '{print $6}'`
      DISK_TYPE=`echo ${LINE}|awk -F\, '{print $7}'`
      DISK_SIZE=`echo ${LINE}|awk -F\, '{print $11}'`

# netapp-rpt-de-carbon-kpi-s_aoxnlfsd02_2021-03-28-07-07-01_cdot-disks.csv
# Sizing in GB
# disk;owner;aggregate;model;type;usable-size;

      echo "CONTROLER=${CONTROLER} DISK_NAME=${DISK_NAME} DISK_VENDOR=${DISK_VENDOR} DISK_MODEL=${DISK_MODEL} DISK_TYPE=${DISK_TYPE} DISK_SIZE=${DISK_SIZE}"


    done # cat .. LINE

  done  # ls -1 *.csv

done  # ls -1 *_$PROCDATE2}_.zip


# ---------------------------------------------------
# NetApp 7mode disks
# From DE 1st: Rename those
# v_decarbon_netapp_cdot_disks_2021-03-16-11-37-54.csv
# netapp-rpt-de-carbon-kpi-s_dcc-new_2021-03-11-16-53-21_cdot-disks.csv
ls -1 ${INDIR}/v_decarbon_netapp_7mode_disks_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST
do
  if [ "${INFILE}" != "" ]; then
# Get only DateTime from filename
    INDATI=`echo ${INFILE}|awk -F\_ '{print $6}'|cut -d\. -f1|grep "2[0-9][0-9][0-9]-[01][0-9]-*"`
    mv ${INFILE} ${INDIR}/netapp-rpt-de-carbon-kpi-s_DE_${INDATI}_7mode-disks.csv
  fi  # INFILE}" != ""
done  # -1 ${INDIR}/v*

# Process netapp 7-mode disks
ls -1 ${INDIR}/netapp*_${PROCDATE}*_7mode-disks.csv 2> /dev/null| while read INFILE REST1
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
  cat ${INFILE}|grep -v ^#|egrep -v 'DISK|-----|owner'|awk -F\; '{print $2}'|sort -u|while read OWNER REST1
  do
    CLUSTERNAME=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
    if [ "${CLUSTERNAME}" != "" ]; then
      COUNTRY=`grep "${OWNER};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      DISKCNT=`cat ${INFILE}|grep -v ^#|egrep -v 'DISK|-----'|grep "${OWNER}"|wc -l`
      MODEL="7mode"
      DISKTYPE=""
      TYPE="HDD"
      USABLE_SIZE=""

      echo "    ${OWNER}=>${CLUSTERNAME}(${COUNTRY}-${OWNER}):${MODEL}(${DISKTYPE}/${TYPE},${USABLE_SIZE})=${DISKCNT}"
      echo "${COUNTRY};${CLUSTERNAME};${OWNER};${MODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks

    else  # CLUSTERNAME}" != ""
      echo "    ${OWNER} unknown"|tee -a ${LOG}
      echo "-;${OWNER};netapp;7mode;" >> ${UNKNOWN_CSV}
    fi  # CLUSTERNAME}" != ""
  done  # cat ${INFILE}
# Move file to PROC(-ed)DIR
  if [ ${MOVE} ]; then
    mv ${INFILE} ${HISTORYDIR}/
  fi
done  # ls -1 ${INDIR}/n*


# ---------------------------------------------------
# EMC-VMAX-disks
ls -1 ${INDIR}/emc*_${PROCDATE}*_vmax-disks.csv 2> /dev/null| while read INFILE REST1
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# 1st ARRAY-loop: Check where (COUNTRY & CLUSTER) ARRAY belongs to
#array ; director:interface:tid ; technology ; vendor_id ; product_id ; serial_id ; total_disk_size_GB
#196801573;DF-1C:C:0;EFD;HITACHI;HUSMM808;2MV2205A;744
#
  cat ${INFILE}|grep -v ^#|grep -v array|awk -F\; '{print $1}'|sort -u|while read ARRAY REST2
  do
    ARRAYNAME=`echo ${ARRAY} |sed "s/'//g"`
# Define to which COUNTRY & CLUSTERNAME an ARRAYNAME belongs
    if [ "${ARRAYNAME}" != "" ]; then
# See if ARRAYNAME is known in CLUSTERFILE
      CLUSTERNAME=`grep "${ARRAYNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
# Check if ARRAYNAME is known. If not, then write to UNKNOWN_CSV
      if [ "${CLUSTERNAME}" != "" ]; then
        COUNTRY=`grep "${ARRAYNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
        cat ${INFILE}|grep -v ^#|grep -v array|grep "${ARRAYNAME};"|cut -d\; -f5|sort -u|while read PROD_ID
        do
#          echo "    ${PROD_ID}"
          DISKCNT=`cat ${INFILE}|grep "${ARRAYNAME};"|grep "${PROD_ID}"|wc -l`
          DISKTECH=`cat ${INFILE}|grep "${PROD_ID}"|cut -d\; -f3|sort -u|head -1`
          USABLE_SIZE=`cat ${INFILE}|grep "${PROD_ID}"|cut -d\; -f7|sort -u|tail -1`
          if [ "${DISKTECH}" = "EFD" ]; then
            TYPE="SSD (${DISKTECH})"
          else
            TYPE="HDD (${DISKTECH})"
          fi
          echo "    ${ARRAYNAME}=>${CLUSTERNAME}(${COUNTRY}-${ARRAYNAME}):${PROD_ID}(${TYPE})=${DISKCNT}"
          echo "${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${PROD_ID};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
        done  # cat ${INFILE} .. read PROD_ID
      else  # CLUSTERNAME}" != ""
        echo "    ${ARRAYNAME} unknown"|tee -a ${LOG}
        echo "-;${ARRAYNAME};emc;vmax;" >> ${UNKNOWN_CSV}
      fi  # CLUSTERNAME}" != ""
    fi  # ARRAYNAME}" != ""
  done  # cat INFILE .. ARRAY

# Move file to PROC(-ed)DIR
  if [ ${MOVE} ]; then
    mv ${INFILE} ${HISTORYDIR}/
  fi
done  # ls -1 *_vmax-disks.csv

# ------------------------------------------------------
# EMC-VNX-disks
ls -1 ${INDIR}/emc*_${PROCDATE}*_vnx-disks.csv 2> /dev/null| while read INFILE REST1
do
  echo "  ${INFILE} ..."|tee -a ${LOG}
# 1st SYSNAME-loop: Check where (COUNTRY & CLUSTER) SYSNAME belongs to
# systemname;disk;model;type;capacity
# (type=tech)
# nlba0vnx09;0_0_0;ST910006CLAR1000;NL SAS;939194
  cat ${INFILE}|grep -v ^#|grep -v systemname|cut -d\; -f1|sort -u|while read SYSNAME REST2
  do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
    if [ "${SYSNAME}" != "" ]; then
      CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
      COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
# See if SYSNAME is known in CLUSTERFILE
      if [ "${CLUSTERNAME}" != "" ]; then
        cat ${INFILE}|grep -v ^#|grep -v systemname|grep "${SYSNAME};"|cut -d\; -f3|sort -u|while read DISKMODEL
        do
#          echo "    ${DISKMODEL}"
          DISKCNT=`cat ${INFILE}|grep "${SYSNAME};"|grep "${DISKMODEL}"|wc -l`
          DISKTECH=`cat ${INFILE}|grep "${DISKMODEL}"|cut -d\; -f4|sort -u|head -1`
          USABLE_SIZE=`cat ${INFILE}|grep "${DISKMODEL}"|cut -d\; -f5|sort -u|tail -1`
          ANSW=`echo "${DISKTECH}"|grep Flash`
          if [ "${ANSW}" != "" ]; then
            TYPE="SSD (${DISKTECH})"
          else
            TYPE="HDD (${DISKTECH})"
          fi
#          echo "      |${DISKMODEL}(${DISKTECH}/${TYPE})=${DISKCNT}"
          echo "      ${SYSNAME}=>${CLUSTERNAME}(${COUNTRY}-${SYSNAME}):${DISKMODEL}/${DISKTECH}(${TYPE})=${DISKCNT}(${USABLE_SIZE})"
          echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
#          echo "${COUNTRY};${CLUSTERNAME};${TECH};${DISKCNT};${TYPE};;" >> ${TMP}.disks
        done  # TECH
      else  # CLUSTERNAME}" != ""
        echo "    ${SYSNAME} (VNX) unknown"|tee -a ${LOG}
        echo "-;${SYSNAME};emc;vnx;" >> ${UNKNOWN_CSV}
      fi  # CLUSTERNAME}" != ""
    fi  # SYSNAME}" != ""
  done  # cat INFILE

# Move file to PROC(-ed)DIR
  if [ ${MOVE} ]; then
    mv ${INFILE} ${HISTORYDIR}/
  fi
done  # ls -1 ${INDIR}/emc*


# EMC-Unity

# From DE 1st: Rename those
# v_decarbon_emc_unity_disks_2021-03-18-11-40-28.csv
# emc-rpt-de-carbon-kpi-s_DE_2021-03-11-16-53-21_unity-disks.csv
ls -1 ${INDIR}/v_decarbon_emc_unity_disks_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST
do
  if [ "${INFILE}" != "" ]; then
# Get only DateTime from filename
    INDATI=`echo ${INFILE}|awk -F\_ '{print $6}'|cut -d\. -f1|grep "2[0-9][0-9][0-9]-[01][0-9]-*"`
    mv ${INFILE} ${INDIR}/emc-rpt-de-carbon-kpi-s_DE_${INDATI}_unity-disks.csv
  fi  # INFILE}" != ""
done  # -1 ${INDIR}/v*


ls -1 ${INDIR}/emc*_${PROCDATE}*_unity-disks.csv 2> /dev/null| while read INFILE REST1
do
  echo "  ${INFILE} ..."|tee -a ${LOG}
  cat ${INFILE}|grep -v ^#|egrep -v "sys_name|de-carbon-kpi-s"|cut -d\; -f1|sort -u|while read SYSNAME REST2
  do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
    if [ "${SYSNAME}" != "" ]; then
      echo "    SYSNAME=${SYSNAME}"
      CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
      COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
# See if SYSNAME is known in CLUSTERFILE
      if [ "${CLUSTERNAME}" != "" ]; then
        echo "    SYSNAME=${SYSNAME}=>CLUSTERNAME=${CLUSTERNAME}"
# sys_name;sys_serialNumber;sys_model;disk_id;disk_model;disk_vendorSize;disk_technology
# DEKHER99U8SSTO;CKM00184200696;Unity 350F;dpe_disk_0;5SRB384C EMC3840;4222124650660;SAS_FLASH_4
        cat ${INFILE}|grep -v ^#|grep -v systemname|grep "${SYSNAME};"|cut -d\; -f5|sort -u|while read DISKMODEL
        do
          DISKCNT=`cat ${INFILE}|grep "${SYSNAME};"|grep "${DISKMODEL}"|wc -l`
          DISKTECH=`cat ${INFILE}|grep "${DISKMODEL}"|cut -d\; -f7|sort -u|head -1`
          USABLE_SIZE=`cat ${INFILE}|grep "${DISKMODEL}"|cut -d\; -f6|sort -u|tail -1`
          echo "      DISKMODEL=${DISKMODEL}:DISKCNT=${DISKCNT}:DISKTECH=${DISKTECH}:USABLE_SIZE=${USABLE_SIZE}"

          ANSW=`echo "${DISKTECH}"|grep FLASH`
          if [ "${ANSW}" != "" ]; then
            TYPE="SSD (${DISKTECH})"
          else
            TYPE="HDD (${DISKTECH})"
          fi
#          echo "      |${DISKMODEL}(${DISKTECH}/${TYPE})=${DISKCNT}"
          echo "      ${SYSNAME}=>${CLUSTERNAME}(${COUNTRY}-${SYSNAME}):${DISKMODEL}/${DISKTECH}(${TYPE})=${DISKCNT}(${USABLE_SIZE})"

          echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks

        done  # TECH
      else  # CLUSTERNAME}" != ""
        echo "    ${SYSNAME} (Unity) unknown"|tee -a ${LOG}
        echo "-;${SYSNAME};emc;unity;" >> ${UNKNOWN_CSV}
      fi  # CLUSTERNAME}" != ""
    fi  # SYSNAME}" != ""
  done  # cat INFILE


done  # -1 ${INDIR}/emc



# -----------------------------------------
# DataDomain DPA
# DPA_ScheduledReport_2021-01-05-09-51-09_disk-config.csv
# When NO PROCDATE, rename with proc-date
# (0.25): Change FileName in DPA-ScheduledReport_DataDomain-Disk-Config-AT_2021-03-15-16-30-30.csv
#  if [ -f ${INDIR}/DPA_ScheduledReport.csv ]; then
#    mv ${INDIR}/DPA_ScheduledReport.csv ${INDIR}/DPA_ScheduledReport_${PROCDATE}_disk-config.csv
#  fi  # DPA_ScheduledReport.csv
#
ls -1 ${INDIR}/DPA-ScheduledReport_*_${PROCDATE}*.csv 2> /dev/null| while read INFILE REST1
do
  echo "  ${INFILE} ..."|tee -a ${LOG}
# 1st SYSNAME-loop: Check where (COUNTRY & CLUSTER) SYSNAME belongs to
# Hostname,Device,Model,Serial Number,Firmware Revision,Size (B),DiskType,ClusterID
# arbueddm01.dcinabox.philips.com,1_1,HITACHI HUS72604CLAR3000,K3GEUZMB,N9CH,2990370979840,SAS,PPH_DCiaB
  cat ${INFILE}|grep -v ^#|grep -v Hostname|cut -d\, -f1|sort -u|while read SYSNAME REST2
  do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
    echo "  SYSNAME=${SYSNAME}"

    if [ "${SYSNAME}" != "" ]; then
# check if SYSNAME <> "globalname" like dds01..03 / ddn01 (from AT)
# If so, then look to the ClusterID part. And find the COUNTRY with this
      GLOBALNAME=`echo ${SYSNAME}|egrep "^ddn|^dds"`
      echo "      GLOBALNAME=${GLOBALNAME}"
      if [ "${GLOBALNAME}" != "" ]; then
        CLUSTERNAME=`grep "${SYSNAME}," ${INFILE}|grep -v ^#|grep -v Hostname|grep -v "Virtual_disk"|cut -d\, -f8|sort -u|head -1`
        COUNTRY=`grep "${GLOBALNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      else
        CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
        COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      fi  # GLOBALNAME}" != ""
      echo "      GLOBALNAME=${GLOBALNAME}=>CLUSTERNAME=${CLUSTERNAME}(COUNTRY=${COUNTRY})"

# See if SYSNAME is known in CLUSTERFILE
      if [ "${CLUSTERNAME}" != "" ]; then
        echo "    CLUSTERNAME=${CLUSTERNAME}"
# Exclude header "Hostname,....
# Also EXcluce "Virtual_disk"
          cat ${INFILE}|grep -v ^#|grep -v Hostname|grep -v "Virtual_disk"|grep "${SYSNAME},"|cut -d\, -f3|sort -u|while read DISKMODEL
        do
#          echo -n "      ${DISKMODEL}="
          DISKCNT=`cat ${INFILE}|grep "${SYSNAME},"|grep "${DISKMODEL}"|wc -l`
# Get disk-TYPE



# Check for " in INFILE.
# When, then some data is having field-sepperator in data.
# Not be able to process.
# Then Skip this part after the "  : 
# BYTE_SIZE=""
          ANSW=`grep '"' ${INFILE}`
          if [ "${ANSW}" != "" ]; then
            DISK_TYPE=""
          else
            DISK_TYPE=`cat ${INFILE}|grep "${DISKMODEL}"|awk -F\, '{print $7}'|sort -u|head -1|sed 's/\r//g'`
          fi
          ANSW=`echo ${DISK_TYPE}|grep SSD`
          if [ "${ANSW}" != "" ]; then
            TYPE="SSD (${DISK_TYPE})"
          else
            TYPE="HDD (${DISK_TYPE})"
          fi

# Check for " in INFILE.
# When, then some data is having field-sepperator in data.
# Not be able to process.
# Then Skip this part after the "  : 
# BYTE_SIZE=""
          ANSW=`grep '"' ${INFILE}`
          if [ "${ANSW}" != "" ]; then
            BYTE_SIZE=""
          else
            BYTE_SIZE=`cat ${INFILE}|grep "${DISKMODEL}"|awk -F\, '{print $6}'|sort -u|head -1|sed 's/\r//g'|sed 's/,.//g'`
          fi
# Removing CR (\r) at the end of the line
          SIZE_GIB=`expr ${BYTE_SIZE} / 1024 / 1024`
          echo "${DISKCNT}(=${TYPE}):${BYTE_SIZE}B"

# Check on NO empty VAR's
          if [ "${COUNTRY}" != "" ] && [ "${CLUSTERNAME}" != "" ] && [ "${SYSNAME}" != "" ] && [ "${DISKMODEL}" != "" ] && [ "${DISKCNT}" != "" ] && [ "${TYPE}" != "" ]; then
            echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${SIZE_GIB};" >> ${TMP}.disks
            echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME} (COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) ; DISKCNT=${DISKCNT} (SIZE_GIB=${SIZE_GIB})" 
          else
            echo "      Some VAR(s) is(are) empty" |tee -a ${LOG}
            echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME}(COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) = DISKCNT=${DISKCNT}(SIZE_GIB=${SIZE_GIB})" |tee -a ${LOG}
          fi
# To pauze when SIZE_GIB=""
          if [ "${SIZE_GIB}" = "" ]; then
            sleep 5
          fi
        done  # DISKMODEL

      else  # CLUSTERNAME}" != ""
        echo "    ${SYSNAME} (DPA) unknown"|tee -a ${LOG}
        echo "-;${SYSNAME};emc;dpa;" >> ${UNKNOWN_CSV}
      fi  # CLUSTERNAME}" != ""

    fi  # SYSNAME}" != ""

  done  # cat INFILE
# Move file to PROC(-ed)DIR
  if [ ${MOVE} ]; then
    mv ${INFILE} ${HISTORYDIR}/
  fi
done  # ls -1 ${INDIR}/DPA*


# DataDomains Disks (from nlxsdp01)
ls -1 ${INDIR}/datadomain_${PROCDATE}_disks-sum.csv 2> /dev/null| while read INFILE REST1
do
  echo "  ${INFILE} ..." |tee -a ${LOG}
  cat ${INFILE}|grep -v ^#|cut -d\; -f1|sort -u|while read SYSNAME REST2
  do
# Find to which COUNTRY & CLUSTERNAME an SYSNAME belongs
# Check is system (SYSNAME) is known. If not, then write to UNKNOWN_CSV
    echo "  SYSNAME=${SYSNAME}"

    if [ "${SYSNAME}" != "" ]; then
      CLUSTERNAME=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $3}'|sort -u|head -1`
      COUNTRY=`grep "${SYSNAME};" ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      echo "    CLUSTERNAME=${CLUSTERNAME}"
# See if SYSNAME is known in CLUSTERFILE
      if [ "${CLUSTERNAME}" != "" ]; then
        cat ${INFILE}|grep -v ^#|grep -v systemname|grep "${SYSNAME};"|while read LINE
        do
# #DD_NAME;SSD;HDD;DRIVES;PROCDATE;REMARKS;
          echo ${LINE}
          DISKMODEL=""
          USABLE_SIZE=""
          TYPE="SSD"
          DISKCNT=`echo ${LINE}|awk -F\; '{print $2}'`
          if [ ${DISKCNT} -gt 0 ]; then
            echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME} (COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) ; DISKCNT=${DISKCNT} (USABLE_SIZE=${USABLE_SIZE})"
            echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
          fi  # DISKCNT} -gt 0
          TYPE="HDD"
          DISKCNT=`echo ${LINE}|awk -F\; '{print $3}'`
          if [ ${DISKCNT} -gt 0 ]; then
            echo "      SYSNAME=${SYSNAME} => CLUSTERNAME=${CLUSTERNAME} (COUNTRY=${COUNTRY}-SYSNAME=${SYSNAME}):DISKMODEL=${DISKMODEL}(TYPE=${TYPE}) ; DISKCNT=${DISKCNT} (USABLE_SIZE=${USABLE_SIZE})"
            echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
          fi  # DISKCNT} -gt 0

        done  # read LINE
      else  # CLUSTERNAME}" != ""
        echo "    ${SYSNAME} (DataDomain) unknown"|tee -a ${LOG}
        echo "-;${SYSNAME};emc;dd;" >> ${UNKNOWN_CSV}
      fi  # CLUSTERNAME}" != ""
    fi  # SYSNAME}" != ""
  done  # read SYSNAME
done  # datadomain_disks




# ========================================================
echo "  ============================================"


# ------ disks --------------------
# Create ALWAY a DISKS_CSV
touch ${DISKS_CSV}
if [ -s ${TMP}.disks ]; then
  echo "    disks"|tee -a ${LOG}
  echo "country; clustername; vendor (emc|hds|netapp); SSD [nr disks]; HDD [nr disks]; date (YYYY-MM-DD);" > ${DISKS_CSV}
#
  cat ${TMP}.disks|grep -v ^#|awk -F\; '{print $2}'|sort -u|while read CLUSTERNAME REST1
# "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" 
# ! CLUSTERNAME NEED to be unique
  do
    if [ "${CLUSTERNAME}" != "" ]; then
      echo "  CLUSTERNAME=${CLUSTERNAME} disks ..."
      TOTAL_SSD_DISKS=0 
      echo "TOTAL_SSD_DISKS=${TOTAL_SSD_DISKS}" > ${TMP}.ssd-disks
      TOTAL_HDD_DISKS=0 
      echo "TOTAL_HDD_DISKS=${TOTAL_HDD_DISKS}" > ${TMP}.hdd-disks

#echo "${COUNTRY};${CLUSTERNAME};${TECH};${DISKCNT};${TYPE};;" >> ${TMP}.disks
      cat ${TMP}.disks|grep -v ^#|grep ${CLUSTERNAME}|sort -u| while read LINE
      do
#echo "LINE=${LINE}"
        COUNTRY=`echo ${LINE}|cut -d\; -f1`
        SYSNAME=`echo ${LINE}|cut -d\; -f3`
        DISKTECH=`echo ${LINE}|cut -d\; -f4`
        DISKCNT=`echo ${LINE}|cut -d\; -f5`
        TYPE=`echo ${LINE}|cut -d\; -f6`
        TYPE_PART=`echo ${LINE}|cut -d\; -f6|cut -d" " -f1`
        USABLE_GB=`echo ${LINE}|cut -d\; -f7`
#      echo "${COUNTRY};${CLUSTERNAME};${DISKTECH};${DISKCNT};${TYPE};${USABLE_GB};" 
        echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${DISKTECH};${DISKCNT};${TYPE};${USABLE_GB};" >> ${DETAILDIR}/${COUNTRY}_${CLUSTERNAME}_${PROCDATE}_disks.csv
        if [ "${TYPE_PART}" = "SSD" ]; then
          TOTAL_SSD_DISKS=`expr ${TOTAL_SSD_DISKS} + ${DISKCNT}`
          echo "      + ${DISKCNT} = ${TOTAL_SSD_DISKS} SSD"
          echo "TOTAL_SSD=${TOTAL_SSD_DISKS}" > ${TMP}.ssd-disks
        else  # ssd
          TOTAL_HDD_DISKS=`expr ${TOTAL_HDD_DISKS} + ${DISKCNT}`
          echo "    + ${DISKCNT} = ${TOTAL_HDD_DISKS} HDD"
          echo "TOTAL_HDD=${TOTAL_HDD_DISKS}" > ${TMP}.hdd-disks
        fi  # ssd
      done  # cat ${TMP}.disks
      COUNTRY=`grep ${CLUSTERNAME} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
      VENDOR=`grep ${CLUSTERNAME} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $4}'|sort -u|head -1`
      if [ "${COUNTRY}" = "" ] || [ "${CLUSTERNAME}" = "" ] || [ "${VENDOR}" = "" ];then
        echo "  ERROR: COUNTRY=${COUNTRY};CLUSTERNAME=${CLUSTERNAME};VENDOR=${VENDOR};"|tee -a ${LOG}
        echo "  ERROR: COUNTRY=${COUNTRY};CLUSTERNAME=${CLUSTERNAME};VENDOR=${VENDOR};"|mailx -s "ERROR in ${PGM} v${VER} at `hostname`" ${MAILTO}
      fi

      if [ "${COUNTRY}" != "" ]; then
        echo "${COUNTRY};${CLUSTERNAME};${VENDOR};`cat ${TMP}.ssd-disks|cut -d\= -f2`;`cat ${TMP}.hdd-disks|cut -d\= -f2`;${PROCDATE};"|tee -a ${DISKS_CSV}
      fi  # COUNTRY}" != ""

      rm ${TMP}.ssd-disks ${TMP}.hdd-disks 
    fi  # CLUSTERNAME}" != ""
  done  # CAT ${TMP}.disks

#  echo "# `date` processed by ${PGM} v${VER} at `hostname`" >> ${DISKS_CSV}

  if [ ${MAIL} ]; then
    date | mailx -a ${DISKS_CSV} -s "Disks CSV file [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  fi  # MAIL
#  rm ${TMP}.disks
fi  # -s ${TMP}.disks

#-----------------------------------
# CLEANUP
if [ ${CLEANUP} ]; then
  ls -1 ${INDIR}/*_${PROCDATE}* 2> /dev/null | while read LINE
  do
    echo "  Moving ${LINE}"|tee -a ${LOG}
    mv ${LINE} ${HISTORYDIR}/
  done  # ls -1
# Remove others from INDIR
  rm ${INDIR}/*.png 2> /dev/null
  rm ${INDIR}/*.jpg 2> /dev/null
fi  # CLEANUP

echo "`date` ${PGM} v${VER} finished"|tee -a ${LOG}
exit 0