#!/bin/sh
# File	: proc_decarbon.sh, 201019
# By	: Maarten.deBoer@Atos.net, 201019
# Subject	: SCript to process input-files with "decarbon" info
#(0.2),201019	: Mod; proc /file. Move file to proc-ed-dir
#(0.3),201102	: Restructured for NetApp
#(0.4),201102	: Added EMC & PROCDATE
#(0.5),201104	: Mod. UNKNOWN_CSV
#(0.6),201104	: Add detail for later usage at upd_
#(0.7),201109	: Add move versions. Added LOG for INFILE. Started with disks
#(0.8),201110	: Cont. with cdot-disks (using TMP.pools & TMP.disks)
PGM=`basename $0|cut -d\. -f1`
VER="0.8"
TMP="/tmp/${PGM}.$$"
BASEDIR="/appl/netapp/decarbon"
LOG="${BASEDIR}/log/${PGM}.log"
INDIR="${BASEDIR}/input"
PROCDIR="${BASEDIR}/proc-ed"
RCFILE="${BASEDIR}/etc/${PGM}.rc"
MAILTO="maarten.deboer@atos.net"
MAIL=""
DETAILDIR="${BASEDIR}/details"

PROCDATE=`date +%Y-%m-%d --set=yesterday`

echo "`date` ${PGM} ${VER} started."|tee -a ${LOG}
touch ${TMP}

if [ -f ${RCFILE} ]; then
  echo "  RCFILE(${RCFILE}) found. Using it ..."|tee -a ${LOG}
  . ${RCFILE}
  sleep 1
fi
CLUSTERFILE="${BASEDIR}/etc/country2systems2cluster2vendor.csv"
POOLS_CSV="${BASEDIR}/data/country-systems-pools_${PROCDATE}.csv"
UNKNOWN_CSV="${BASEDIR}/data/unknown-systems_${PROCDATE}.csv"
DISKS_CSV="${BASEDIR}/data/country-systems-disks_${PROCDATE}.csv"
echo "BASEDIR=${BASEDIR}"
echo "CLUSTERFILE=${CLUSTERFILE}"
echo "DETAILDIR=${DETAILDIR}"
echo "DISKS_CSV=${DISKS_CSV}"
echo "INDIR=${INDIR}"
echo "POOLS_CSV=${POOLS_CSV}"
echo "UNKNOWN_CSV=${UNKNOWN_CSV}"
echo "PROCDIR=${PROCDIR}"
echo "PROCDATE=${PROCDATE}"
sleep 1

if [ ! -f ${CLUSTERFILE} ]; then
  echo "  NO CLUSTER-file (${CLUSTERFILE}) found. Exiting ..."|tee -a ${LOG}
  exit 4
fi  
if [ ! -d ${INDIR} ]; then
  echo "  NO in(put)-dir (${INDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 5
fi  
if [ ! -d ${PROCDIR} ]; then
  echo "  NO proc-ed-dir (${PROCDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 6
fi  
if [ ! -d ${DETAIL} ]; then
  echo "  NO detail-dir (${DETAILDIR}) found. Exiting ..."|tee -a ${LOG}
  exit 7
fi  

touch ${TMP}
if [ ! -f ${UNKNOWN_CSV} ]; then
  echo ";OWNER;" >> ${UNKNOWN_CSV}
fi

# Find file's to be processed
# From TODAY
# Put the TB's from known / unknown in a TMP-file

# NetApp-versions
ls -1 ${INDIR}/n*_${PROCDATE}*_versions.csv| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${PROCDIR}/
done  # ls -1 *_versions.csv


# NetApp-7mode (legacy)
ls -1 ${INDIR}/n*_${PROCDATE}*_7mode-aggrs.csv| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# nlnaf25;Aggregate;total;used;avail;
  cat ${INFILE}|grep -v ^#|grep -v Aggregate|awk -F\; '{print $1,$2,$3}'|while read SYSNAME AGGRNAME SIZE_GB REST
  do
    if [ "${SYSNAME}" != "" ] && [ `grep ${SYSNAME} ${CLUSTERFILE}` ]; then
# SYSNAME = found in CLUSTERFILE
      CLUSTERNAME=`grep ${SYSNAME} ${CLUSTERFILE}|awk -F\; '{print $3}'`
      COUNTRY=`grep ${SYSNAME} ${CLUSTERFILE}|awk -F\; '{print $1}'`
      echo "    ${SYSNAME} => ${CLUSTERNAME} (${COUNTRY}) : ${SIZE_GB}GB"
      echo "${COUNTRY};${CLUSTERNAME};${SYSNAME};${AGGRNAME};legacy;${SIZE_GB};" >> ${TMP}.pools
    else
      echo "    ${SYSNAME} unknown"|tee -a ${LOG}
      echo "-;${SYSNAME};" >> ${UNKNOWN_CSV}
    fi  # grep ${OWNER}
  done  # INFILE
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${PROCDIR}/
done  # ls -1 *_7mode-aggrs.csv


# NetApp-cDOT-aggr's(pools)
ls -1 ${INDIR}/n*_${PROCDATE}*_cdot-aggrs.csv| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# FILE: aggregate;storage-type;aggregate-type;owner-name;size;usedsize;physical-used;
  cat ${INFILE}|grep -v ^#|grep -v aggregate|awk -F\; '{print $1,$2,$3,$4,$5}'|while read AGGRNAME STOR_TYPE AGGR_TYPE OWNER SIZE_GB REST
  do
    if [ "${OWNER}" != "" ] && [ `grep ${OWNER} ${CLUSTERFILE}` ]; then
# OWNER = found in CLUSTERFILE
      CLUSTERNAME=`grep ${OWNER} ${CLUSTERFILE}|awk -F\; '{print $3}'`
      COUNTRY=`grep ${OWNER} ${CLUSTERFILE}|awk -F\; '{print $1}'`
      echo "    ${OWNER} => ${CLUSTERNAME} (${COUNTRY})"

      if [ "${AGGR_TYPE}" = "ssd" ]; then
        echo "${COUNTRY};${CLUSTERNAME};${OWNER};${AGGRNAME};ssd;${SIZE_GB};" >> ${TMP}.pools
      else
        echo "${COUNTRY};${CLUSTERNAME};${OWNER};${AGGRNAME};${AGGR_TYPE};${SIZE_GB};" >> ${TMP}.pools
      fi  # ssd
    else
      echo "    ${OWNER} unknown"|tee -a ${LOG}
      echo "-;${OWNER};" >> ${UNKNOWN_CSV}
    fi  # grep ${OWNER}
  done  # cat INFILE (*_cdot-aggrs.csv)
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${PROCDIR}/
done  # ls -1 *_cdot-aggrs.csv


# EMC-VMAX-pools
ls -1 ${INDIR}/emc*_${PROCDATE}*_vmax-pools.csv| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# array ; tech ; pool_name ; owner ; pool_usable ; pool_used ; pool_physical ; data_ratio ; vp_ratio ; overall_ratio
  cat ${INFILE}|grep -v ^#|grep -v array|awk -F\; '{print $1,$2,$3,$5}'|while read ARRAY TECH POOLNAME USABLE_GB REST
  do
    ARRAYNAME=`echo ${ARRAY} |sed "s/'//g"`
#      echo "    ${ARRAYNAME}"
    if [ "${ARRAYNAME}" != "" ] && [ `grep ${ARRAYNAME} ${CLUSTERFILE}` ]; then
      CLUSTERNAME=`grep ${ARRAYNAME} ${CLUSTERFILE}|awk -F\; '{print $3}'`
      COUNTRY=`grep ${ARRAYNAME} ${CLUSTERFILE}|awk -F\; '{print $1}'`
      echo "    ${ARRAYNAME} => ${CLUSTERNAME} (${COUNTRY})"

      SIZE_GB=`echo ${USABLE_GB}|cut -d\. -f1`

      if [ "${TECH}" = "ssd" ]; then
        echo "${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${POOLNAME};ssd;${SIZE_GB};" >> ${TMP}.pools
      else
        echo "${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${POOLNAME};${TECH};${SIZE_GB};" >> ${TMP}.pools
      fi  # ssd
    else
      echo "    ${ARRAYNAME} unknown"|tee -a ${LOG}
      echo "-;${ARRAYNAME};" >> ${UNKNOWN_CSV}
    fi
  done  # INFILE
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${PROCDIR}/
done  # ls -1 *_vmax-pools.csv


# NetApp(cDOT-)disks
ls -1 ${INDIR}/n*_${PROCDATE}*_cdot-disks.csv| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
# disk;owner;aggregate;model;type;usable-size;
  cat ${INFILE}|grep -v ^#|grep -v disk|awk -F\; '{print $4,$2}'|sort -u|while read DISKMODEL OWNER REST1
  do
    DISKCNT=`grep -c ${DISKMODEL} ${INFILE}`
    TYPE=`grep ${DISKMODEL} ${INFILE}|awk -F\; '{print $5}'|sort -u|head -1`
    USABLE_SIZE=`grep ${DISKMODEL} ${INFILE}|awk -F\; '{print $6}'|sort -u|head -1`
# Check is system (OWNER) is known. If not, then write to UNKNOWN_CSV
    if [ "${OWNER}" != "" ] && [ `grep ${OWNER} ${CLUSTERFILE}` ]; then
      CLUSTERNAME=`grep ${OWNER} ${CLUSTERFILE}|awk -F\; '{print $3}'`
      COUNTRY=`grep ${OWNER} ${CLUSTERFILE}|awk -F\; '{print $1}'`
      echo "    ${OWNER}=>${CLUSTERNAME}(${COUNTRY}):${DISKMODEL}=${DISKCNT}(${TYPE}:${USABLE_SIZE})"
      echo "${COUNTRY};${CLUSTERNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" >> ${TMP}.disks
    else  # grep OWNER
      echo "    ${OWNER} unknown"|tee -a ${LOG}
      echo "-;${OWNER};" >> ${UNKNOWN_CSV}
    fi  # grep OWNER
  done  # cat INFILE
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${PROCDIR}/
done  # ls -1 *_cdot-disks.csv


# EMC-VMAX-disks
ls -1 ${INDIR}/emc*_${PROCDATE}*_vmax-disks.csv| while read INFILE REST
do
  echo "   ${INFILE} ..."|tee -a ${LOG}
#array ; tech ; disk_size_GB
#196801573;EFD;800
  cat ${INFILE}|grep -v ^#|grep -v array|awk -F\; '{print $2,$1}'|sort -u|while read TECH ARRAY REST2
  do
    ARRAYNAME=`echo ${ARRAY} |sed "s/'//g"`
    DISKCNT=`grep -c ${TECH} ${INFILE}`
    if [ "${TECH}" = "EFD" ]; then
      TYPE="SSD"
    else
      TYPE="HDD"
    fi
echo "|${TECH}|${ARRAY}|${ARRAYNAME}|${TYPE}|"
# Check is system (OWNER) is known. If not, then write to UNKNOWN_CSV
    if [ "${ARRAYNAME}" != "" ] && [ `grep ${ARRAYNAME} ${CLUSTERFILE}` ]; then
      CLUSTERNAME=`grep ${ARRAYNAME} ${CLUSTERFILE}|awk -F\; '{print $3}'`
      COUNTRY=`grep ${ARRAYNAME} ${CLUSTERFILE}|awk -F\; '{print $1}'`
      echo "    ${ARRAYNAME}=>${CLUSTERNAME}(${COUNTRY}):${TYPE}=${DISKCNT}"
      echo "${COUNTRY};${CLUSTERNAME};${TECH};${DISKCNT};${TYPE};;" >> ${TMP}.disks
# '{print $2,$3,$4,$5,$6}'|while read CLUSTERNAME DISKMODEL DISKCNT TYPE USABLE_GB REST2
    else  # grep OWNER
      echo "    ${ARRAYNAME} unknown"|tee -a ${LOG}
      echo "-;${ARRAYNAME};" >> ${UNKNOWN_CSV}
    fi  # grep OWNER
  done  # cat INFILE
# Move file to PROC(-ed)DIR
  mv ${INFILE} ${PROCDIR}/
done  # ls -1 *_vmax-disks.csv



echo "  -------------------------------------------"

# 2nd part when TMP.pools is not empty
# TMP: ${COUNTRY};${CLUSTERNAME};${OWNER};${AGGRNAME};${AGGR_TYPE};${SIZE_GB};
# TMP: ${COUNTRY};${CLUSTERNAME};${ARRAYNAME};${POOLNAME};${TECH};${SIZE_GB};
echo "  2nd part."|tee -a ${LOG}

# ------ pools / aggr's --------------------
if [ -s ${TMP}.pools ]; then
  echo "# country; system(cluster / HA)name; vendor ; SSD [GiB]; HDD [GiB]; Legay(7mode) [GiB]; " > ${POOLS_CSV}
  TOTAL_SSD=0 
  echo "TOTAL_SSD=${TOTAL_SSD}" > ${TMP}.ssd
  TOTAL_HDD=0 
  echo "TOTAL_OLD=${TOTAL_OLD}" > ${TMP}.old
# Legaycy = OLD
  TOTAL_OLD=0 
  echo "TOTAL_HDD=${TOTAL_HDD}" > ${TMP}.hdd
  cat ${TMP}.pools|sort -u|grep -v ^#|awk -F\; '{print $2}'|while read CLUSTER REST1
  do
    echo "  ${CLUSTER}"
    TOTAL_SSD=0 
    echo "TOTAL_SSD=${TOTAL_SSD}" > ${TMP}.ssd
    TOTAL_HDD=0 
    echo "TOTAL_HDD=${TOTAL_HDD}" > ${TMP}.hdd
    TOTAL_OLD=0 
    echo "TOTAL_OLD=${TOTAL_OLD}" > ${TMP}.old
    cat ${TMP}.pools|sort -u|grep -v ^#|grep ${CLUSTER}|awk -F\; '{print $2,$3,$4,$5,$6}'|while read CLUSTERNAME SYSTEMNAME POOLNAME TYPE SIZE_GB REST2
    do
#      echo "    ${TYPE} ${SIZE_GB} "
      echo "${CLUSTERNAME};${SYSTEMNAME};${POOLNAME};${TYPE};${SIZE_GB}" >> ${DETAILDIR}/${CLUSTERNAME}_${PROCDATE}_pools.csv
      if [ "${TYPE}" = "ssd" ]; then
        TOTAL_SSD=`expr ${TOTAL_SSD} + ${SIZE_GB}`
        echo "      + ${SIZE_GB}GB = ${TOTAL_SSD} SSD (${POOLNAME})"
        echo "TOTAL_SSD=${TOTAL_SSD}" > ${TMP}.ssd
      else  # ssd
        if [ "${TYPE}" = "legacy" ]; then
          TOTAL_OLD=`expr ${TOTAL_OLD} + ${SIZE_GB}`
          echo "    + ${SIZE_GB}GB = ${TOTAL_OLD} OLD (${POOLNAME})"
          echo "TOTAL_OLD=${TOTAL_OLD}" > ${TMP}.old
        else  # legacy
          TOTAL_HDD=`expr ${TOTAL_HDD} + ${SIZE_GB}`
          echo "    + ${SIZE_GB}GB = ${TOTAL_HDD} HDD (${POOLNAME})"
          echo "TOTAL_HDD=${TOTAL_HDD}" > ${TMP}.hdd
        fi  # legacy
      fi  # ssd
    done  # cat TMP
    COUNTRY=`grep ${CLUSTER} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
    VENDOR=`grep ${CLUSTER} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $4}'|sort -u|head -1`
    echo "${COUNTRY};${CLUSTER};${VENDOR};`cat ${TMP}.ssd|cut -d\= -f2`;`cat ${TMP}.hdd|cut -d\= -f2`;`cat ${TMP}.old|cut -d\= -f2`;"|tee -a ${POOLS_CSV}
    rm ${TMP}.ssd ${TMP}.hdd ${TMP}.old
  done  # cat TMP.pools


  echo "# `date` ${PGM} v${VER} at `hostname`" >> ${POOLS_CSV}

  if [ ${MAIL} ]; then
    date | mailx -a ${POOLS_CSV} -s "Pools/Aggrs CSV file [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  fi  # MAIL
  rm ${TMP}.pools 
fi  # -s ${TMP}.pools


# ------ disks --------------------
if [ -s ${TMP}.disks ]; then
  echo "# country; system(cluster / HA)name; vendor ; SSD [# disks]; HDD [# disks]; " > ${DISKS_CSV}
# "${COUNTRY};${CLUSTERNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_SIZE};" 
  cat ${TMP}.disks|sort -u|grep -v ^#|awk -F\; '{print $2}'|while read CLUSTER REST1
  do
    echo "  ${CLUSTER}"
    TOTAL_SSD_DISKS=0 
    echo "TOTAL_SSD_DISKS=${TOTAL_SSD_DISKS}" > ${TMP}.ssd-disks
    TOTAL_HDD_DISKS=0 
    echo "TOTAL_HDD_DISKS=${TOTAL_HDD_DISKS}" > ${TMP}.hdd-disks

    cat ${TMP}.disks|sort -u|grep -v ^#|grep ${CLUSTER}|awk -F\; '{print $2,$3,$4,$5,$6}'|while read CLUSTERNAME DISKMODEL DISKCNT TYPE USABLE_GB REST2
    do
      echo "${CLUSTERNAME};${DISKMODEL};${DISKCNT};${TYPE};${USABLE_GB}" >> ${DETAILDIR}/${CLUSTERNAME}_${PROCDATE}_disks.csv
      if [ "${TYPE}" = "SSD" ]; then
        TOTAL_SSD_DISKS=`expr ${TOTAL_SSD_DISKS} + ${DISKCNT}`
        echo "      + ${DISKCNT} = ${TOTAL_SSD_DISKS} SSD"
        echo "TOTAL_SSD=${TOTAL_SSD_DISKS}" > ${TMP}.ssd-disks
      else  # ssd
        TOTAL_HDD_DISKS=`expr ${TOTAL_HDD_DISKS} + ${DISKCNT}`
        echo "    + ${DISKCNT} = ${TOTAL_HDD_DISKS} HDD"
        echo "TOTAL_HDD=${TOTAL_HDD_DISKS}" > ${TMP}.hdd-disks
      fi  # ssd
    done  # cat ${TMP}.disks
    COUNTRY=`grep ${CLUSTER} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|head -1`
    VENDOR=`grep ${CLUSTER} ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $4}'|sort -u|head -1`
    echo "${COUNTRY};${CLUSTER};${VENDOR};`cat ${TMP}.ssd-disks|cut -d\= -f2`;`cat ${TMP}.hdd-disks|cut -d\= -f2`;"|tee -a ${DISKS_CSV}
    rm ${TMP}.ssd-disks ${TMP}.hdd-disks 
  done  # CAT ${TMP}.disks

  if [ ${MAIL} ]; then
    date | mailx -a ${DISKS_CSV} -s "Disks CSV file [${PGM} v${VER}]" ${MAILTO}
    echo "  Mailed to ${MAILTO}"|tee -a ${LOG}
  fi  # MAIL
  rm ${TMP}.disks
fi  # -s ${TMP}.disks


echo "`date` ${PGM} ${VER} finished"|tee -a ${LOG}
exit 0