#!/bin/bash # File : upd_decarbon.sh # By : Maarten.deBoer, 201019 # Subject : Script to update the decarbon website #(0.2),201020 : Add unknown systems in 1st row #(0.3),201103 : Mod. to (new) structure #(0.4),201103 : Upd. some text #(0.5),201104 : Upd. with newest file in DATADIR #(0.6),201104 : Added DETAIL, SYSNAME -> CLUSTERNAME #(0.7),201105 : Added overview of INPUT-dir (input.html) #(0.8),201105 : Add HTML_TOP(), HTML_FOOTER() #(0.9),201105 : Add mailing some pages to extern #(0.10),201110 : Added disks (& POOLSFILE * DISKSFILE) #(0.11),201110 : Add DISKMODELINFO #(0.12),201111 : Add. EXTRA wiki & USAGE() #(0.13),201112 : Mod. to # SSD : HDD #(0.14),201116 : Mod. disks 1st, 2nd aggr sizes #(0.15),201116 : Added TITLE2, Mod. NO AGGR only disks (numbers) #(0.16),201126 : Added AVG_PWR for SSD & HDD PGM="`basename $0|cut -d\. -f1`" VER="0.15" TMP="/tmp/${PGM}.$$" BASEDIR="/appl/netapp/decarbon" LOG="${BASEDIR}/log/${PGM}.log" RCFILE="${BASEDIR}/etc/${PGM}.rc" UP_DATI=`date +%Y-%m-%d-%H-%M-%S` MAIL=1 MAILTO="maarten.deboer@atos.net" TXTFILE="${BASEDIR}/etc/${PGM}.txt" #HTMLMAILTO="atos@agrarix.it" # # Google: "Average ssd power consumption" # HDD = 9.1 W # SSD = 0.06 W (60 mW) AVG_PWR_SSD_MW="60" AVG_PWR_HDD_MW="9100" DATADIR="${BASEDIR}/data" ETCDIR="${BASEDIR}/etc" DETAILDIR="${BASEDIR}/details" INDIR="${BASEDIR}/input" DISKMODELINFO="${BASEDIR}/etc/diskmodel2info.csv" PAGEDIR="/appl/netapp/html/decarbon" TITLE="Storage DeCarbonize Dashboard" TITLE2="Number of disks" #PROCDATE=`date +%Y-%m-%d --set=yesterday` HTML_TOP() { HTML_PAGE="${1}" touch ${HTML_PAGE} echo "" > ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo "">> ${HTML_PAGE} echo "">> ${HTML_PAGE} echo "${TITLE}" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} #echo "" >> ${PAGE} echo "" >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} #echo " " >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo "
" >> ${PAGE} echo " " >> ${HTML_PAGE} echo " ${TITLE} " >> ${HTML_PAGE} echo "
" >> ${HTML_PAGE} if [ "${TITLE2}" != "" ]; then echo "
${TITLE2} " >> ${HTML_PAGE} fi # TITLE2 # echo "
" >> ${HTML_PAGE} # echo " For storage environments in country => " >> ${HTML_PAGE} # echo " " >> ${HTML_PAGE} echo "
" >> ${HTML_PAGE} echo " At `hostname` " >> ${HTML_PAGE} echo "
" >> ${HTML_PAGE} echo "
" >> ${HTML_PAGE} } HTML_FOOTER() { HTML_PAGE="${1}" touch ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo " " >> ${HTML_PAGE} echo "" >> ${HTML_PAGE} } USAGE() { echo "Usage: ${PGM} [options]" echo "Options : " echo " -h | --help : This help" echo " -r | --rc : change Rcfile (${RCFILE})" echo " --indir : change INputDIRectory (${INDIR})" echo " -V : Version" } while [ ${#} -gt 0 ] do case "${1}" in -h | --help) USAGE; exit 1 ;; -r | --rc) RCFILE="${2}"; shift ;; --indir) INDIR="${2}"; shift ;; -x) set -x ;; -V) echo "Version=${VER}" ; exit 1 ;; *) echo "Option ${1} not found." ; USAGE; exit 1 ;; esac shift done echo "`date` ${PGM} ${VER} started."|tee -a ${LOG} touch ${TMP} if [ -f ${RCFILE} ]; then echo " RCFILE(${RCFILE}) found. Using it ..."|tee -a ${LOG} . ${RCFILE} fi CLUSTERFILE="${BASEDIR}/etc/country2systems2cluster2vendor.csv" PAGE="${PAGEDIR}/index.html" #DATAFILE="${BASEDIR}/data/country-systems_${PROCDATE}.csv" #UNKNOWNFILE="${BASEDIR}/data/unknown-systems_${PROCDATE}.csv" echo "PGM=${PGM}" echo "VER=${VER}" echo "LOG=${LOG}" echo "CLUSTERFILE=${CLUSTERFILE}" echo "DATADIR=${DATADIR}" #echo "DATAFILE=${DATAFILE}" echo "DETAILDIR=${DETAILDIR}" echo "INDIR=${INDIR}" echo "PAGEDIR=${PAGEDIR}" echo "PAGE=${PAGE}" #echo "PROCDATE=${PROCDATE}" echo "TITLE=${TITLE}" #echo "UNKNOWNFILE=${UNKNOWNFILE}" sleep 1 if [ ! -f ${CLUSTERFILE} ]; then echo " NO NO CLUSTER-file (${CLUSTERFILE}) found. Exiting ..."|tee -a ${LOG} exit 4 fi if [ ! -d ${DATADIR} ]; then echo " NO DATA-dir (${DATADIR}) found. Exiting ..."|tee -a ${LOG} exit 5 fi if [ ! -d ${DETAILDIR} ]; then echo " NO DETAIL-dir (${DETAILDIR}) found. Exiting ..."|tee -a ${LOG} exit 6 fi if [ ! -f ${DISKMODELINFO} ]; then echo " NO DISKMODEL-INFO (${DISKMODELINFO}) found. Exiting ..."|tee -a ${LOG} exit 7 fi # "dummy" country-systems-pools touch ${DATADIR}/country-systems-pools_YYYY-MM-DD-HH-MM-SS.csv touch ${DATADIR}/country-systems-disks_YYYY-MM-DD-HH-MM-SS.csv HTML_TOP ${PAGE} echo "" >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} # Countries echo " Country (names) ..." cat ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|while read COUNTRY REST do # echo " ${COUNTRY} " # selecte LAST POOLS-file available DISKSFILE=`grep "^${COUNTRY};" ${DATADIR}/country-systems-disks*.csv|cut -d\: -f1|sort -u|tail -1` # POOLSFILE=`grep "^${COUNTRY};" ${DATADIR}/country-systems-pools*.csv|cut -d\: -f1|sort -u|tail -1` # Get PROCDATE /country PROCDATE=`echo ${DISKSFILE}|cut -d\_ -f2|cut -d\. -f1` if [ "${PROCDATE}" != "" ]; then PROCDATE="(${PROCDATE})" fi echo " " >> ${PAGE} done # ${CLUSTERFILE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " Country info ..." cat ${CLUSTERFILE}|grep -v ^#|awk -F\; '{print $1}'|sort -u|while read COUNTRY REST do echo " ${COUNTRY} " echo " " >> ${PAGE} done # ${CLUSTERFILE} echo " " >> ${PAGE} echo "
" >> ${PAGE} echo " Unknown" >> ${PAGE} echo " " >> ${PAGE} echo "
${COUNTRY} ${PROCDATE}" >> ${PAGE} echo "
" >> ${PAGE} # Check if a unknown-file has some information cat ../data/unknown-systems_*.csv|grep -v ^#|awk -F\; '{print $2}'|sort -u|while read LINE do echo " ${LINE}

" >> ${PAGE} done # cat echo " 		" >> ${PAGE} # selecte LAST file available # selecte LAST file available POOLSFILE=`grep "^${COUNTRY};" ${DATADIR}/country-systems-pools*.csv|cut -d\: -f1|sort -u|tail -1` DISKSFILE=`grep "^${COUNTRY};" ${DATADIR}/country-systems-disks*.csv|cut -d\: -f1|sort -u|tail -1` #echo "POOLSFILE=${POOLSFILE}" #echo "DISKSFILE=${DISKSFILE}" PROCDATE=`echo ${DISKSFILE}|cut -d\_ -f2|cut -d\. -f1` if [ "${DISKSFILE}" != "" ]; then # --------------------------------------- # SYStem # country; system(cluster / HA)name; vendor ; SSD [#]; HDD [#]; " cat ${DISKSFILE}|grep "^${COUNTRY};"|sort -u|grep -v ^#|sort|awk -F\; '{print $2,$3}'|while read CLUSTERNAME VENDOR REST do echo " ${CLUSTERNAME} (${VENDOR}) [${PROCDATE}]" # When a SYStemNAME, then process for html-PAGE if [ "${CLUSTERNAME}" != "" ]; then # update DETAILs if [ -f ${DETAILDIR}/${COUNTRY}_${CLUSTERNAME}_${PROCDATE}_disks.csv ]; then DPAGE="${PAGEDIR}/${COUNTRY}_${CLUSTERNAME}.html" HTML_TOP ${DPAGE} # nlnaf100;nlnaf100-01;n01a0601;hdd;100402 echo " ${DETAILDIR}/${COUNTRY}_${CLUSTERNAME}_${PROCDATE}_disks.csv" echo "

Details of ${CLUSTERNAME} disks :

" >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} # DISKS cat ${DETAILDIR}/${COUNTRY}_${CLUSTERNAME}_${PROCDATE}_disks.csv|sort -u|grep -v ^#|while read LINE do CLUSTERNAME=`echo ${LINE}|cut -d\; -f2` DISKMODEL=`echo ${LINE}|cut -d\; -f3` DISKCNT=`echo ${LINE}|cut -d\; -f4` TYPE=`echo ${LINE}|cut -d\; -f5` USABLE_GB=`echo ${LINE}|cut -d\; -f6` echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} if [ "${TYPE}" = "ssd" ] || [ "${TYPE}" = "SSD" ]; then echo " " >> ${DPAGE} else if [ "${TYPE}" = "legacy" ]; then echo " " >> ${DPAGE} else echo " " >> ${DPAGE} fi fi # TYPE echo " " >> ${DPAGE} echo " " >> ${DPAGE} done # cat .csv echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo " " >> ${DPAGE} echo "
COUNTRY CLUSTERNAME DISKMODEL DISKCNT TYPE USABLE_GB
${COUNTRY} ${CLUSTERNAME} ${DISKMODEL} ${DISKCNT} ${TYPE} ${TYPE} ${TYPE} ${USABLE_GB}
Name of country" >> ${DPAGE} echo " Name of location / cluster Disk model / tech Nomber of disks Type of disks [sdd, hdd, hybrid, legacy] Size in GiB
" >> ${DPAGE} HTML_FOOTER ${DPAGE} fi # DETAILs _disks if [ "${HTMLMAILTO}" != "" ]; then cat ${DPAGE}|mailx -s "html:${COUNTRY}_${CLUSTERNAME}.html" ${HTMLMAILTO} echo " Mailed ${COUNTRY}_${CLUSTERNAME}.html to ${HTMLMAILTO}"|tee -a ${LOG} fi SSD_CNT=`grep "${COUNTRY};" ${DISKSFILE}|grep "${CLUSTERNAME};"|grep -v ^#|sort -u|head -1|awk -F\; '{print $4}'` if [ "${SSD_CNT}" = "" ]; then SSD_CNT=0 fi HDD_CNT=`grep "${COUNTRY};" ${DISKSFILE}|grep "${CLUSTERNAME};"|grep -v ^#|sort -u|head -1|awk -F\; '{print $5}'` if [ "${HDD_CNT}" = "" ]; then HDD_CNT=0 fi echo " S:${SSD_CNT} H:${HDD_CNT} " echo " " >> ${PAGE} echo " " >> ${PAGE} # Calculate PCT for "bar"-graph TTL_CNT=`expr ${SSD_CNT} + ${HDD_CNT}` if [ ${TTL_CNT} -gt 0 ]; then PCT1=`expr ${SSD_CNT} \* 100 / ${TTL_CNT}` PCT2=`expr ${HDD_CNT} \* 100 / ${TTL_CNT}` else PCT1=0 PCT2=0 fi echo " ${PCT1}% : ${PCT2}% / #${TTL_CNT}" SSD_PWR_W=`expr ${SSD_CNT} \* ${AVG_PWR_SSD_MW} / 1000` HDD_PWR_W=`expr ${HDD_CNT} \* ${AVG_PWR_HDD_MW} / 1000` echo " H:${HDD_PWR_W}Watt | S:${SSD_PWR_W}W" V="" if [ "${VENDOR}" = "netapp" ]; then V="N" fi if [ "${VENDOR}" = "emc" ]; then V="E" fi if [ "${VENDOR}" = "hds" ]; then V="H" fi #echo "VENDOR=${VENDOR} V=${V}" # Count (SSD | HDD-)disks # NR_SSD=`grep "${COUNTRY};" ${DISKSFILE}|grep "${CLUSTERNAME};"|sort -u|grep -v ^#|sort|awk -F\; '{print $4}'` # NR_HDD=`grep "${COUNTRY};" ${DISKSFILE}|grep "${CLUSTERNAME};"|sort -u|grep -v ^#|sort|awk -F\; '{print $5}'` if [ -f ${PAGEDIR}/${COUNTRY}_${CLUSTERNAME}.html ]; then echo " " >> ${PAGE} else echo " " >> ${PAGE} fi echo " " >> ${PAGE} echo " " >> ${PAGE} # echo " " >> ${PAGE} echo " " >> ${PAGE} echo "
${V}:${CLUSTERNAME}: ${V}:${CLUSTERNAME}: ${SSD_CNT}(${PCT1}%) =${SSD_PWR_W}W ${HDD_CNT}(${PCT2}%) =${HDD_PWR_W}W ${OLD_TB}
" >> ${PAGE} fi # CLUSTERNAME <> "" done # CLUSTERNAME fi # POOLSFILE <> "" echo "
" >> ${PAGE} # EXTRA echo "
" >> ${PAGE} echo "" >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} #echo " " >> ${PAGE} echo " " >> ${PAGE} echo "
EXTRA: Wiki inputdir Grafana(ext)
" >> ${PAGE} echo "
" >> ${PAGE} echo "" >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} #echo " " >> ${PAGE} #echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} echo " " >> ${PAGE} AVG_PWR_SSD_MW="60" AVG_PWR_HDD_MW="9100" echo " " >> ${PAGE} echo "
Legenda:
E = H = N =
Mumber of disks SSD HDD Legacy(N:7mode) TiB Number of disks between ()
Avg pwr consumption /disk ${AVG_PWR_SSD_MW}mW ${AVG_PWR_HDD_MW}mW
" >> ${PAGE} # When (explanation)TXT-file if [ -f ${TXTFILE} ]; then echo "
" >> ${PAGE} echo " " >> ${PAGE} cat ${TXTFILE}|while read LINE do echo "${LINE}
" >> ${PAGE} done # TXTFILE echo " " >> ${PAGE} fi # TXTFILE HTML_FOOTER ${PAGE} # INPUTdir-overview IPAGE="${PAGEDIR}/inputdir.html" HTML_TOP ${IPAGE} echo " " >> ${IPAGE} echo " " >> ${IPAGE} echo " " >> ${IPAGE} ls -1tr ${INDIR}|while read LINE do echo " " >> ${IPAGE} echo " " >> ${IPAGE} echo " " >> ${IPAGE} done # ls -1 echo "
FILENAME in input-dir: " >> ${IPAGE} echo "
${LINE}
" >> ${IPAGE} HTML_FOOTER ${IPAGE} if [ "${HTMLMAILTO}" != "" ]; then cat ${PAGE}|mailx -s "html:decarbon.html" ${HTMLMAILTO} cat ${IPAGE}|mailx -s "html:inputdir.html" ${HTMLMAILTO} echo " Mailed HTML-pages to ${HTMLMAILTO}"|tee -a ${LOG} fi rm ${TMP} echo "`date` ${PGM} ${VER} finished."|tee -a ${LOG} exit 0