# from email.MIMEBase import MIMEBase
from email.mime.base import MIMEBase
# from email.MIMEMultipart import MIMEMultipart
from email.mime.multipart import MIMEMultipart
# from email.MIMEText import MIMEText
from email.mime.text import MIMEText
from email import encoders
from datetime import date
from datetime import datetime as dt
import base64
import ConfigParser
import csv
import errno
import getpass
import glob
import logging
import logging.handlers
import os
import shlex
import smtplib
import socket
import subprocess
import sys
import time
import zipfile
class LogUserContextFilter(logging.Filter):
"""Filter class that will inject user id into the each log record"""
def filter(self, record):
record.user = getpass.getuser()
return True
# Hostname
hostname = socket.gethostname()
# Solution name
script_name = os.path.basename(os.path.splitext(__file__)[0])
# Solution directory
script_dir = os.path.dirname(os.path.realpath(__file__))
# Solution logs directory
log_dir = os.path.join(script_dir, 'log')
# Creating log directory if does not exists
if not os.path.exists(log_dir):
try:
os.makedirs(log_dir)
except OSError as e:
if e.errno != os.errno.EEXIST:
print 'ERROR - Unable to create log directory'
print 'ERROR - Logs directory full path: %s' % (log_dir)
print ('ERROR - Please check if you have the right permissions to'
' create it')
sys.exit(1)
# Logging
log_filename = "%s_%s.log" % (hostname, script_name)
log_file = os.path.join(log_dir, log_filename)
log_file_date_fmt = '%Y-%m-%d'
# Daily log autorotation
if os.path.exists(log_file):
log_mtime = os.path.getmtime(log_file)
log_mdate_str = time.strftime(log_file_date_fmt,
time.localtime(log_mtime))
log_mdate = dt.strptime(log_mdate_str, log_file_date_fmt).date()
if log_mdate < date.today():
rotated_log_file = log_file + "." + log_mdate_str
os.rename(log_file, rotated_log_file)
old_log_files = glob.glob(log_file + '.*')
if len(old_log_files) > 6:
oldest_log_file = sorted(old_log_files)[0]
os.remove(oldest_log_file)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Create user context filter
user_filter = LogUserContextFilter()
logger.addFilter(user_filter)
fformatter = logging.Formatter('%(asctime)s - %(levelname)-7s -'
' %(funcName)-13s - %(message)s - %(user)s')
cformatter = logging.Formatter('%(levelname)-7s - %(message)s')
fh = logging.FileHandler(log_file, mode='a')
fh.setLevel(logging.DEBUG)
fh.setFormatter(fformatter)
logger.addHandler(fh)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(cformatter)
logger.addHandler(ch)
try:
import mysql.connector
except ImportError:
logger.error("Python mysql-connector must be installed for running "
"this script")
logger.error("Please read the README.txt file or if it is missing "
"please contact")
logger.error("Atos Romania's Cloud Development team: "
"dl-ro-cloud-dev@atos.net")
sys.exit(127)
class ConfigINItoDict(ConfigParser.ConfigParser):
"""
ConfigParser.ConfigParser subclass with to_dict method added in order
to convert the current ConfigParser object with a read config file to a
a dictionary
"""
def to_dict(self):
"""
Converts ConfigINItoDict object to a dictionary
:return: the dictionary representation of the ConfigINItoDict object
"""
config = dict(self._sections)
for section in config:
config[section] = dict(self._defaults, **config[section])
config[section].pop('__name__', None)
return config
def check_empty_options(name, section, options, log_lvl):
"""
Checks if the options of the section are empty strings.
Returns a list with the names of the empty options
:param name: section name
:param section: dict representation of the section
:param options: list with options to be checked
:param log_lvl: default log_level
:returns bool
"""
valid_opts = True
empty_opts = list()
for opt in options:
if section[opt] == '':
empty_opts.append(opt)
if len(empty_opts) > 0:
valid_opts = False
log_lvl = 40
log_msg = ('[%s] Please set the right values for the following '
'options: %s' % (name, ', '.join(empty_opts)))
logger.log(log_lvl, log_msg)
return valid_opts
def check_config(loaded_conf, standalone=False):
"""
Validates the config sections and thier related config option by checking
if they respect the config file formatting rules
:param loaded_conf: The dict representation of the .ini config file
:param standalone: Used to set the appropiate log level if is called only
to check the config option (standalone = True).
Default means that the function is used as part of the
the scripts normal workflow
:returns bool
"""
valid_config = True
# Country Codes lists
allowed_country_codes = (
# Austria
'AT',
# Belgium
'BE',
# Switzerland
'CH',
# Germany
'DE',
# Denmark
'DK',
# Spain
'ESP',
# Finland
'FIN',
# France
'FR',
# Hungary
'HUN',
# Netherlands
'NL',
# Sweden
'SWE',
# United Kingdom
'UK'
)
collector_data_opts = {
'm_opts': set(['country']),
'o_opts': set(['export_dir']),
}
ocum_database_opts = {
'm_opts': set(['host', 'user', 'password', 'port']),
'o_opts': None
}
scp_xfer_opts = {
'm_opts': set(['enable', 'ssh_key']),
'o_opts': set(['host', 'user', 'data_dir'])
}
email_opts = {
'm_opts': set(['server', 'from', 'to']),
'o_opts': None,
}
email_xfer_opts = {
'm_opts': set(['enable']),
'o_opts': set(['server', 'from', 'to'])
}
conf_struct = {
'mandatory_sections': {
'collector_data': collector_data_opts,
'OCUM_database': ocum_database_opts,
'email': email_opts
},
'optional_sections': {
'scp_transfer': scp_xfer_opts,
'email_transfer': email_xfer_opts
}
}
log_lvl, default_log_level = 10, 10
if standalone:
log_lvl, default_log_level = 20, 20
logger.log(log_lvl, 'Started validating config')
logger.log(log_lvl, 'Config: %s' % (loaded_conf))
logger.log(log_lvl, 'Validating mandatory sections')
log_msg = ("Successfully validated mandatory sections: %s" %
(', '.join(conf_struct['mandatory_sections'])))
loaded_sections = loaded_conf.keys()
if not (set(conf_struct['mandatory_sections'].keys()) <=
set(loaded_sections)):
log_lvl = 40
missing_sections = (set(conf_struct['mandatory_sections'].keys()) -
set(loaded_sections))
log_msg = ('Missing mandatory section(s): %s' %
', '.join(missing_sections))
valid_config = False
logger.log(log_lvl, log_msg)
log_lvl = default_log_level
loaded_mandatory_sections = (
set(loaded_sections) & set(conf_struct['mandatory_sections'].keys())
)
for section in loaded_mandatory_sections:
log_lvl = default_log_level
logger.log(log_lvl, '[%s] - Validating options' % (section))
loaded_opts = set(loaded_conf[section].keys())
logger.log(log_lvl, '[%s] - Validating mandatory options' %
(section))
log_msg = ('[%s] - Successfully validated mandatory options' %
(section))
if not (conf_struct['mandatory_sections'][section]['m_opts'] <=
set(loaded_opts)):
log_lvl = 40
valid_config = False
missing_mandatory_opts = (
conf_struct['mandatory_sections'][section]['m_opts'] -
set(loaded_opts))
logger.log(log_lvl, '[%s] - Missing mandatory options: %s' %
(section, ', '.join(missing_mandatory_opts)))
continue
logger.log(log_lvl,
'[%s] - Successfully validated mandatory options' %
(section)
)
e_opts = check_empty_options(
section, loaded_conf[section],
conf_struct['mandatory_sections'][section]['m_opts'],
log_lvl
)
if not e_opts:
valid_config = e_opts
if e_opts and section in ['OCUM_database']:
try:
user = base64.b64decode(loaded_conf[section]['user'])
except TypeError as e:
if e.message.message == 'Incorrect padding':
log_lvl = 40
valid_config = False
logger.log(log_lvl, '[%s] - Invalid user format provided.'
% (section))
logger.log(log_lvl,
'[%s] - Please use the ro-cloud-pw-marshall '
'tool to properly set the user' % (section))
logger.log(log_lvl,
'[%s] - Please refer to the '
'ro-cloud-cmdb-tools documentation for more '
'details' % (section))
try:
user = base64.b64decode(loaded_conf[section]['password'])
except TypeError as e:
if e.message.message == 'Incorrect padding':
log_lvl = 40
valid_config = False
logger.log(log_lvl, '[%s] - Invalid password format '
'provided.' % (section))
logger.log(log_lvl,
'[%s] - Please use the ro-cloud-pw-marshall '
'tool to properly set the password' % (section)
)
logger.log(log_lvl,
'[%s] - Please refer to the '
'ro-cloud-cmdb-tools documentation for more '
'details' % (section))
if (section == 'collector_data' and
loaded_conf[section]['country'] not in allowed_country_codes):
log_lvl = 40
valid_config = False
log_msg = ('[%s] - Invalid value: country = %s. Supported '
'country codes: %s' %
(section, loaded_conf[section]['country'],
', '.join(allowed_country_codes)))
logger.log(log_lvl, log_msg)
if conf_struct['optional_sections'] is None:
return valid_config
log_lvl = default_log_level
loaded_opt_sections = (set(loaded_sections) &
set(conf_struct['optional_sections'].keys()))
if (len(loaded_opt_sections) > 0):
logger.log(log_lvl, 'Optional sections: %s' %
(', '.join(loaded_opt_sections)))
for section in loaded_opt_sections:
log_lvl = default_log_level
logger.log(log_lvl, '[%s] - Validating mandatory options' %
(section))
loaded_opts = set(loaded_conf[section].keys())
if not (conf_struct['optional_sections'][section]['m_opts'] <=
set(loaded_opts)):
log_lvl = 40
valid_config = False
missing_mandatory_opts = (
conf_struct['optional_sections'][section]['m_opts'] -
set(loaded_opts))
logger.log(log_lvl, '[%s] - Missing mandatory options: %s' %
(section, ', '.join(missing_mandatory_opts)))
continue
e_opts = check_empty_options(
section, loaded_conf[section],
conf_struct['optional_sections'][section]['m_opts'],
log_lvl
)
if not e_opts:
valid_config = e_opts
if (section == 'scp_transfer' and
loaded_conf[section]['enable'].lower() == 'true'):
if not (conf_struct['optional_sections'][section]['o_opts'] <=
set(loaded_conf[section].keys())):
log_lvl = 40
valid_config = False
missing_mandatory_opts = (
conf_struct['optional_sections'][section]['o_opts'] -
set(loaded_opts))
logger.log(log_lvl,
'[%s] - Missing mandatory options: %s' %
(section, ', '.join(missing_mandatory_opts)))
e_opts = check_empty_options(
section, loaded_conf[section],
conf_struct['optional_sections'][section]['o_opts'],
log_lvl
)
if not e_opts:
valid_config = e_opts
if (section == 'email_transfer' and
loaded_conf[section]['enable'].lower() == 'true'):
if not (conf_struct['optional_sections'][section]['o_opts'] <=
set(loaded_conf[section].keys())):
log_lvl = 40
valid_config = False
missing_mandatory_opts = (
conf_struct['optional_sections'][section]['o_opts'] -
set(loaded_opts))
logger.log(log_lvl,
'[%s] - Missing mandatory options: %s' %
(section, ', '.join(missing_mandatory_opts)))
e_opts = check_empty_options(
section, loaded_conf[section],
conf_struct['optional_sections'][section]['o_opts'],
log_lvl
)
if not e_opts:
valid_config = e_opts
log_lvl = default_log_level
if valid_config:
logger.log(log_lvl,
'[%s] - Successfully validated mandatory options' %
(section))
else:
logger.log(log_lvl, 'No optional sections in config file')
return valid_config
def runCmd(cmd):
"""
Runs specific CLI commands by creating it's own subprocess. SSH issued
commands can be used easily
:param cmd: the command string to be executed
:return: (str, str, int)
"""
args = shlex.split(cmd)
proc = subprocess.Popen(args)
stdoutdata, stderrdata = proc.communicate()
ret_code = proc.returncode
return stdoutdata, stderrdata, ret_code
def scp_files(files, host, user, destination_directory, ssh_key):
"""
Copies files over SCP.
:param files: list of file paths to be copied
:param host: the destination host name or IP address
:param user: the destination's user used for copying the files
:param destination_directory: The remote host where we want to save the
files
:param ssh_key: The user's ssh key path
:return int: The scp command return code
"""
scpCmd = ('/usr/bin/scp -i ' + ssh_key + ' ' + ' '.join(files) + ' ' +
user + '@' + host + ':' + destination_directory)
logger.debug('Copying file(s) to: %s' % (host))
logger.debug('File(s): %s' % (', '.join(files)))
(stdout, stderr, ret_code) = runCmd(scpCmd)
result_msg = "Copied files(s) to %s:%s" % (host, destination_directory)
if ret_code != 0:
result_msg = "Unable to copy files to %s " % (host)
logger.debug(result_msg)
return ret_code
def send_mail(send_from, send_to, subject, body, body_type='plain',
files=[], server='localhost'):
"""
Send an email to a list of recipients. Optional files can be added.
:param send_from: the sender e-mail address
:param type: str
:param send_to: a list of email addresses (the recipients list)
:param type: list
:param subject: The email subject
:param type: str
:param body: Long string for detailing the purpose of the email
:param type: str
:param body_type: body content type. Defaults to `plain`.
body_type='html' could be used too.
:param type: str
:param files: list of file paths to be attached
:param type: list
:param server: The mail server. Defaults to localhost
:param type: str
"""
message = MIMEMultipart("alternative")
message['From'] = send_from
message['To'] = ','.join(send_to)
message['Subject'] = subject.decode('utf-8')
body_content = "%s"
if body_type == 'html':
body_content = "\n\n\n%s\n\n"
message.attach(MIMEText(body_content % (body), body_type, 'utf-8'))
for f in files:
part = MIMEBase('application', 'base64')
part.set_payload(open(f, 'rb').read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="%s"' %
os.path.basename(f))
message.attach(part)
smtp = smtplib.SMTP(server)
smtp.sendmail(send_from, send_to, message.as_string())
smtp.quit()
def load_config(config_file_path):
"""
Loads the config file and returns its dict like representation.
Exits with error if the [connection] section is not present
:param config_file_path: The config file path
:return: the config dictionary
"""
logger.debug("Started loading the config file")
logger.debug("Config file path: %s" % (config_file_path))
config_handler = ConfigINItoDict()
dataset = config_handler.read(config_file_path)
if len(dataset) == 0:
logger.error("Failed to open/find file %s" % (config_file_path))
sys.exit(1)
config = config_handler.to_dict()
logger.debug("Config loaded: %s" % (config))
return config
def build_report(report, mysql_cursor, mysql_query, csv_headers,
data_dir, country_code):
"""
Builds the csv report file for the passed mysql query. Returns True if
report has been generated and the csv report file path OR False if report
has not been generated and None for the file_path.
:param report: The report name
:param mysql_cursor: The MySQL connection's cursor
:param mysql_query: The MySQL query for building the report
:param csv_headers: The list of strings representing the header for the
exported csv file. IMPORTANT: The list must have the
exact same headers as described in the mysql_query and
their order will dictate the column's order too
:param data_dir: The path of the directory where we want to save the
csv report file.
:param country_code: The country code for which we generate the report
:return boolean, string: The status of the report generation and the
report file size.
Return values:
1. True, CSV file path string
2. False, None
"""
logger.debug("Getting report data for %s" % (report))
current_date = date.today()
report_date = current_date.strftime('%Y%m%d')
csv_file_name = "%s_%s_%s_%s.csv" % (country_code, hostname, report,
report_date)
csv_file_path = os.path.join(data_dir, csv_file_name)
logger.debug("%s: Executing query" % (report))
mysql_cursor.execute(mysql_query)
logger.debug("%s: Fetching query results" % (report))
data = list()
for row in mysql_cursor:
data.append(row)
if mysql_cursor.rowcount == 0:
return False, None
logger.debug("%s: Fetched %d results" % (report, len(data)))
logger.debug("%s: Creating csv data file" % (report))
with open(csv_file_path, 'wb') as csv_file:
dw = csv.DictWriter(csv_file, csv_headers, delimiter=';',
lineterminator='\r\n', quoting=csv.QUOTE_ALL)
dw.writerow(dict(zip(csv_headers, csv_headers)))
dw.writerows(data)
return True, csv_file_path
def archive_files(files, archive_name, archive_directory):
"""
Creates an archive with the provided list of file paths.
If archive directory is provided the archive will be saved under that
specific path, if None then it will be created where the source file
is located
:param files: List of file paths to be archived
:param archive_name: The archive name
:param archive_directory: The path where we want to save the archive
:return archive path
"""
archive_file_name = archive_name + ".zip"
archive_file_path = os.path.join(archive_directory, archive_file_name)
logger.debug("Archive file path: %s" % (archive_file_path))
archive = zipfile.ZipFile(archive_file_path, mode='w')
for f in files:
file_name = os.path.basename(f)
archive.write(f, arcname=file_name)
logger.debug("Added %s to archive" % (file_name))
os.remove(f)
logger.debug("Deleted %s" % (file_name))
archive.close()
logger.debug("Finised creating archive: %s" % (archive_file_path))
return archive_file_path
def cdot_collect_db_data(conf):
"""
Creates the cdot reports csv files by querying the OCUM database,
archives and sends them to the central inventory database.
:param conf: the config dictionary
"""
# TODO @cnesa: add support for clustom export dir
logger.debug("Started running the data collection")
disks_header = ['cluster', 'node', 'diskname', 'disktype', 'size',
'labeledsize', 'pool', 'interface', 'rpm', 'partno',
'firmware_revision', 'serial_number', 'reportdate']
disks_sql = """\
SELECT
cluster.name AS cluster,
disk.homeNodeName AS node,
disk.name AS diskname,
disk.raidPosition AS disktype,
ROUND(disk.totalBytes / POW(1024, 3), 6) AS size,
ROUND(disk.totalBytes / POW(1024, 3), 6) AS labeledsize,
disk.pool AS pool,
disk.interfaceType AS interface,
disk.rpm AS rpm,
disk.model AS partno,
disk.firmwareRevision AS firmware_revision,
disk.serialNumber AS serial_number,
CURDATE() AS reportdate
FROM
netapp_model.disk
JOIN
netapp_model.cluster ON cluster.objid = disk.clusterId
WHERE
disk.objState = 'LIVE' AND disk.containerTypeRaw != 'remote';
"""
aggrcapacity_header = ['cluster', 'node', 'aggregate', 'aggrcap',
'aggrused', 'aggrsnapcap', 'aggrsnapused',
'mirrored', 'reportdate']
aggrcapacity_sql = """\
SELECT
cluster.name AS cluster,
node.name AS node,
aggregate.name AS aggregate,
ROUND(aggregate.sizeTotal / POW(1024, 3), 6) AS aggrcap,
ROUND(aggregate.sizeUsed / POW(1024, 3), 6) AS aggrused,
ROUND(aggregate.snapshotSizeTotal / POW(1024, 3),
6) AS aggrsnapcap,
ROUND(aggregate.snapshotSizeUsed / POW(1024, 3),
6) AS aggrsnapused,
IF(aggregate.mirrorStatus = 'MIRRORED',
'yes',
'no') AS mirrored,
CURDATE() AS reportdate
FROM
aggregate
JOIN netapp_model.cluster ON cluster.objid = aggregate.clusterId
JOIN netapp_model.node ON node.objid = aggregate.nodeId
WHERE
aggregate.objState = 'LIVE';
"""
storagesystems_header = ['cluster', 'node', 'nodepartner', 'manufacturer',
'model', 'ontapversion', 'ocumhost',
'serialnumber', 'systemid', 'adminip', 'firmware',
'nfs', 'cifs', 'snapmirror', 'multistore', 'fcp',
'iscsi', 'snapvaultprimary', 'snapvaultsecondary',
'is_metrocluster', 'mc_type', 'mc_partner',
'reportdate']
storagesystems_sql = """\
SELECT
(SELECT cluster.name
FROM cluster
WHERE cluster.objid=partner1.clusterId) AS cluster,
partner1.name AS node,
(SELECT partner2.name
FROM node AS partner2
WHERE partner1.partnerNodeId=partner2.objid) AS nodepartner,
partner1.vendor AS manufacturer,
partner1.model AS model,
SUBSTRING_INDEX(partner1.version,':',1) AS ontapversion,
(SELECT @@hostname) AS ocumhost,
partner1.serialNumber AS serialnumber,
partner1.nvramId AS systemid,
(SELECT cluster.managementIp
FROM cluster
WHERE cluster.objid=partner1.clusterId) AS adminip,
partner1.cpuFirmwareRelease AS firmware,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='nfs' AND lic.ownerNodeId=partner1.objid) AS nfs,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='cifs' AND lic.ownerNodeId=partner1.objid) AS cifs,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='snapmirror' AND lic.ownerNodeId=partner1.objid
) AS snapmirror,
1 AS multistore,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='fcp' AND lic.ownerNodeId=partner1.objid) AS fcp,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='iscsi' AND lic.ownerNodeId=partner1.objid
) AS iscsi,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='snapvault'
AND lic.ownerNodeId=partner1.objid) AS snapvaultprimary,
(SELECT COUNT(*)
FROM license AS lic
WHERE lic.package='snapvault' AND lic.ownerNodeId=partner1.objid
) AS snapvaultsecondary,
(SELECT cluster.isMetrocluster
FROM cluster
WHERE cluster.objid=partner1.clusterId
) AS 'is_metrocluster',
(SELECT cluster.mtConfigurationType
FROM cluster
WHERE cluster.objid=partner1.clusterId
) AS 'mc_type',
(SELECT cluster.name
FROM cluster
WHERE cluster.mtDrPartnerClusterId=partner1.clusterId
) AS 'mc_partner',
CURDATE() AS reportdate
FROM node AS partner1
WHERE partner1.objState = 'LIVE';
"""
svm_header = ['cluster', 'svm', 'svm_type', 'svm_subtype', 'ipspace',
'ipaddress', 'dns_enabled', 'dns_domains', 'dns_servers',
'svm_protocols', 'svmstorage', 'reportdate']
svm_sql = """\
SELECT
(SELECT
cluster.name
FROM
cluster
WHERE
cluster.objID = vserver.clusterId) AS cluster,
vserver.name AS svm,
vserver.typeRaw AS svm_type,
vserver.subtypeRaw AS svm_subtype,
(SELECT
network_ip_space.name
FROM
network_ip_space
WHERE
network_ip_space.objid = vserver.networkIpSpaceId) AS ipspace,
(SELECT
GROUP_CONCAT(address)
FROM
network_lif
WHERE
network_lif.vserverId = vserver.objid) AS ipaddress,
vserver.allowedProtocols AS svm_protocols,
vserver.dnsEnabled AS dns_enabled,
vserver.dnsDomainNames AS dns_domains,
vserver.dnsServers AS dns_servers,
(SELECT
GROUP_CONCAT(name)
FROM
volume
WHERE
volume.vserverId = vserver.objid) AS svmstorage,
CURDATE() AS reportdate
FROM
vserver
WHERE
vserver.objState = 'LIVE'
AND vserver.type IN ('DATA' , 'NODE', 'ADMIN');
"""
volcapacity_header = ['cluster', 'svm', 'aggregate', 'volume', 'volcap',
'volused', 'volsnapcap', 'volsnapused',
'snapshot_policy', 'is_vserver_root', 'is_clone',
'clone_parent_name', 'svm_parent_name', 'reportdate']
volcapacity_sql = """\
SELECT
(SELECT
cluster.name
FROM
netapp_model.cluster
WHERE
cluster.objid = vol.clusterId) AS cluster,
(SELECT
vserver.name
FROM
netapp_model.vserver
WHERE
vserver.objid = vol.vserverId) AS svm,
(SELECT
aggregate.name
FROM
netapp_model.aggregate
WHERE
aggregate.objid = vol.aggregateId) AS aggregate,
vol.name AS volume,
ROUND(vol.size / POW(1024, 3), 6) AS volcap,
ROUND(vol.sizeUsed / POW(1024, 3), 6) AS volused,
ROUND(vol.snapshotReserveSize / POW(1024, 3),
6) AS volsnapcap,
ROUND(vol.sizeUsedBySnapshots / POW(1024, 3),
6) AS volsnapused,
(SELECT
snap_policy.name
FROM
netapp_model.snapshot_policy AS snap_policy
WHERE
snap_policy.objid = vol.snapshotPolicyId) AS snapshot_policy,
vol.isVserverRoot AS is_vserver_root,
IFNULL((SELECT IF(ISNULL(volume_self_relationship.cloneParentId),
'0',
'1')
FROM
volume_self_relationship
WHERE
volume_self_relationship.objid = vol.objId
AND volume_self_relationship.objstate = 'LIVE'),
'0') AS is_clone,
(SELECT
volume.name
FROM
volume
RIGHT JOIN
netapp_model.volume_self_relationship ON volume.objid = volume_self_relationship.cloneParentId
WHERE
volume_self_relationship.objid = vol.objId
AND volume.objstate = 'LIVE') AS clone_parent_name,
(SELECT
vserver.name
FROM
volume
RIGHT JOIN
netapp_model.volume_self_relationship ON volume.objid = volume_self_relationship.cloneParentId
RIGHT JOIN
netapp_model.vserver ON vserver.objid = volume.vserverId
WHERE
volume_self_relationship.objid = vol.objId
AND volume.objstate = 'LIVE') AS svm_parent_name,
CURDATE() AS reportdate
FROM
netapp_model.volume AS vol
WHERE
vol.objState = 'LIVE';
"""
qtreecapacity_header = ['cluster', 'svm', 'volume', 'qtree', 'qtreecap',
'qtreeused', 'cap_soft_limit', 'qtree_status',
'quota_type', 'file_quota_size', 'file_quota_used',
'file_quota_soft_limit', 'reportdate']
qtreecapacity_sql = """\
SELECT
cluster.name AS cluster,
vserver.name AS svm,
volume.name AS volume,
qt.name AS qtree,
ROUND(qtree_quota.diskLimit / POW(1024, 2), 6) AS qtreecap,
ROUND(qtree_quota.diskUsed / POW(1024, 2), 6) AS qtreeused,
ROUND(qtree_quota.softDiskLimit / POW(1024, 2), 6) AS cap_soft_limit,
qt.statusRaw AS qtree_status,
qtree_quota.quotaType AS quota_type,
qtree_quota.fileLimit AS file_quota_size,
qtree_quota.fileUsed AS file_quota_used,
qtree_quota.softFileLimit AS file_quota_soft_limit,
CURDATE() AS reportdate
FROM
netapp_model.qtree AS qt
JOIN
netapp_model.qtree_quota ON qtree_quota.qtreeId = qt.objid
JOIN
netapp_model.vserver ON vserver.objid = qt.vserverId
JOIN
netapp_model.volume ON volume.objid = qt.volumeId
JOIN
netapp_model.cluster ON cluster.objid = qt.clusterId
WHERE
qt.objState = 'LIVE' AND qt.name != ''
AND qtree_quota.objState = 'LIVE';
"""
nfsshares_header = ['cluster', 'svm', 'volume', 'qtree', 'export_policy',
'junction_active', 'junction_path', 'protocol',
'client_info', 'read_access', 'write_access',
'reportdate']
nfsshares_sql = """\
SELECT
cluster.name AS cluster,
vserver.name AS svm,
volume.name AS volume,
qtree.name AS qtree,
export_policy.name AS export_policy,
volume.isJunctionActive AS junction_active,
volume.junctionPath AS junction_path,
export_rule.accessProtocol AS protocol,
export_rule.clientMatch AS client_info,
export_rule.roRule AS read_access,
export_rule.rwRule AS write_access,
CURDATE() AS reportdate
FROM
netapp_model.export_policy
JOIN
netapp_model.export_rule ON
export_rule.exportPolicyId = export_policy.objid
JOIN
netapp_model.qtree ON qtree.exportPolicyId = export_policy.objid
JOIN
netapp_model.volume ON qtree.volumeId = volume.objid
JOIN
netapp_model.vserver ON volume.vserverId = vserver.objid
JOIN
netapp_model.cluster ON vserver.clusterId = cluster.objid
WHERE
volume.objState = 'LIVE'
AND export_policy.objState = 'LIVE'
AND export_rule.objState = 'LIVE'
AND qtree.objState = 'LIVE';
"""
cifsshares_header = ['cluster', 'svm', 'volume', 'qtree', 'cifs_server',
'share_name', 'path', 'client_info', 'share_access',
'reportdate']
cifsshares_sql = """\
SELECT
cluster.name AS cluster,
vserver.name AS svm,
IF(ISNULL(cifs_shares.volumeId),
REPLACE(SUBSTRING_INDEX(cifs_shares.path, '/', 2),
'/',
''),
(SELECT
name
FROM
netapp_model.volume
WHERE
volume.objid = cifs_shares.volumeId)) AS volume,
(SELECT
name
FROM
qtree
WHERE
qtree.objid = cifs_shares.qtreeId) AS qtree,
vserver.cifsServer AS cifs_server,
cifs_shares.name AS share_name,
cifs_shares.path AS path,
cifs_share_acl.userOrGroup AS client_info,
cifs_share_acl.permission AS share_access,
CURDATE() AS reportdate
FROM
netapp_model.cifs_share AS cifs_shares
JOIN
netapp_model.cifs_share_acl ON cifs_share_acl.cifsShareId = cifs_shares.objid
JOIN
netapp_model.vserver ON cifs_shares.vserverId = vserver.objid
JOIN
netapp_model.cluster ON vserver.clusterId = cluster.objid
WHERE
cluster.objState = 'LIVE'
AND vserver.objState = 'LIVE'
AND cifs_shares.objState = 'LIVE'
AND cifs_share_acl.objState = 'LIVE';
"""
luninfo_header = ['cluster', 'svm', 'volume', 'qtree', 'igroup_name',
'initiator', 'lun_path', 'lun_id', 'lun_serial',
'naa_number', 'os_type', 'protocol', 'allocated_size',
'used_size', 'lun_online', 'lun_mapped',
'isSpaceReservationEnabled', 'isSpaceAllocEnabled',
'reportdate']
luninfo_sql = """\
SELECT
cluster.name AS cluster,
vserver.name AS svm,
volume.name AS volume,
qtree.name AS qtree,
igroup.name AS igroup_name,
igroup_initiator.name AS initiator,
lun.path AS lun_path,
lun_map.lun AS lun_id,
lun.serialNumber AS lun_serial,
IF((cluster.versionGeneration >= 9) or (cluster.versionGeneration >= 8
AND cluster.versionMajor >= 3),
CONCAT('naa.600a0980', HEX(lun.serialNumber)),
CONCAT('naa.60a98000', HEX(lun.serialNumber))) AS naa_number,
lun.multiProtocolType AS os_type,
igroup.type AS protocol,
ROUND(lun.size / POW(1024, 3), 6) AS allocated_size,
ROUND(lun.sizeUsed / POW(1024, 3), 6) AS used_size,
lun.isOnline AS lun_online,
lun.isMapped AS lun_mapped,
lun.isSpaceReservationEnabled,
lun.isSpaceAllocEnabled,
CURDATE() AS reportdate
FROM
netapp_model.lun
JOIN
netapp_model.vserver ON lun.vserverId = vserver.objid
JOIN
netapp_model.cluster ON lun.clusterId = cluster.objid
JOIN
netapp_model.volume ON lun.volumeId = volume.objid
LEFT JOIN
netapp_model.qtree ON lun.qtreeId = qtree.objid
LEFT JOIN
netapp_model.lun_map ON lun_map.lunId = lun.objid
LEFT JOIN
netapp_model.igroup ON lun_map.igroupId = igroup.objid
LEFT JOIN
netapp_model.igroup_initiator ON igroup_initiator.igroupId = igroup.objid
WHERE
lun.objState = 'LIVE';
"""
storagefficiency_header = ['cluster', 'svm', 'volume', 'volused', 'dedup',
'dedupperc', 'compr', 'comprperc', 'reportdate']
storagefficiency_sql = """\
SELECT
cluster.name AS cluster,
vserver.name AS svm,
vol.name AS volume,
ROUND(vol.sizeUsed / POW(1024, 3), 6) AS volused,
ROUND(vol.deduplicationSpaceSaved / POW(1024, 3), 6) AS dedup,
vol.percentageDeduplicationSpaceSaved AS dedupperc,
ROUND(vol.compressionSpaceSaved / POW(1024, 3), 6) AS compr,
vol.percentageCompressionSpaceSaved AS comprperc,
CURDATE() AS reportdate
FROM
netapp_model.volume AS vol
JOIN
netapp_model.vserver ON vserver.objid = vol.vserverId
JOIN
netapp_model.cluster ON cluster.objid = vol.clusterId
WHERE
vol.objState = 'LIVE';
"""
snapshotpolicies_header = ['cluster', 'svm', 'policy_name',
'policy_enabled', 'snapshots_count',
'sched_prefix', 'sched_job_name', 'sched_type',
'cron_day_of_month', 'cron_day_of_week',
'cron_hour', 'cron_minute', 'cron_month',
'interval_days', 'interval_hours',
'interval_minutes', 'interval_seconds',
'svm_type', 'reportdate']
snapshotpolicies_sql = """\
SELECT
cluster.name AS cluster,
vserver.name AS svm,
snap_policy.name AS policy_name,
snap_policy.isEnabled AS policy_enabled,
snap_sched.count AS snapshots_count,
snap_sched.prefix AS sched_prefix,
jobs.name AS sched_job_name,
jobs.scheduleTypeRaw AS sched_type,
jobs.cronDayOfMonth AS cron_day_of_month,
jobs.cronDayOfWeek AS cron_day_of_week,
jobs.cronHour AS cron_hour,
jobs.cronMinute AS cron_minute,
jobs.cronMonth AS cron_month,
jobs.intervalDays AS interval_days,
jobs.intervalHours AS interval_hours,
jobs.intervalMinutes AS interval_minutes,
jobs.intervalSeconds AS interval_seconds,
vserver.typeRaw AS svm_type,
CURDATE() AS reportdate
FROM
netapp_model.snapshot_policy_schedule AS snap_sched
JOIN netapp_model.snapshot_policy AS snap_policy
ON snap_policy.objid = snap_sched.snapshotPolicyId
JOIN netapp_model.cluster
ON cluster.objid = snap_policy.clusterId
JOIN netapp_model.job_schedule AS jobs
ON jobs.objid = snap_sched.jobScheduleId
JOIN netapp_model.snapshot_policy_vserver_relationship AS policy_svm
ON policy_svm.objid = snap_sched.snapshotPolicyId
LEFT JOIN netapp_model.vserver
ON vserver.objid = policy_svm.vserverId
WHERE
snap_sched.objState = 'LIVE';
"""
replicationinfo_header = ['relation_type', 'src_cluster', 'src_svm',
'src_volume', 'src_location', 'dest_location',
'dest_cluster', 'dest_svm', 'dest_node',
'dest_volume', 'policy_name', 'lag_time',
'last_transfer_size_kb',
'last_transfer_duration', 'last_transfer_type',
'schedule_type', 'cron_day_of_month',
'cron_day_of_week', 'cron_hour', 'cron_minute',
'cron_month', 'interval_days', 'interval_hours',
'interval_minutes', 'interval_seconds',
'reportdate']
replicationinfo_sql = """\
SELECT
(CASE
WHEN sm_policy.typeRaw IN ('vault' , 'mirror_vault')
THEN 'snapvault'
WHEN sm_policy.typeRaw IN ('async_mirror' , 'sync_mirror')
THEN 'snapmirror'
ELSE sm_policy.typeRaw
END) AS relation_type,
sm.sourceCluster AS src_cluster,
sm.sourceVserver AS src_svm,
sm.sourceVolume AS src_volume,
sm.sourceLocation AS src_location,
sm.destinationLocation AS dest_location,
cluster.name AS dest_cluster,
vserver.name AS dest_svm,
node.name AS dest_node,
volume.name AS dest_volume,
sm_policy.name AS policy_name,
sm.lagTime AS lag_time,
ROUND(sm.lastTransferSize / POW(1024, 1), 3) AS last_transfer_size_kb,
sm.lastTransferDuration AS last_transfer_duration,
sm.lastTransferTypeRaw AS last_transfer_type,
job_sched.scheduleType AS schedule_type,
job_sched.cronDayOfMonth AS cron_day_of_month,
job_sched.cronDayOfWeek AS cron_day_of_week,
job_sched.cronHour AS cron_hour,
job_sched.cronMinute AS cron_minute,
job_sched.cronMonth AS cron_month,
job_sched.intervalDays AS interval_days,
job_sched.intervalHours AS interval_hours,
job_sched.intervalMinutes AS interval_minutes,
job_sched.intervalSeconds AS interval_seconds,
CURDATE() AS reportdate
FROM
netapp_model.snap_mirror AS sm
JOIN
netapp_model.cluster ON cluster.objid = sm.clusterId
LEFT JOIN
netapp_model.vserver ON vserver.objid = sm.vserverId
LEFT JOIN
netapp_model.node ON node.objid = sm.nodeId
LEFT JOIN
netapp_model.volume ON volume.objid = sm.volumeId
JOIN
netapp_model.snap_mirror_policy AS sm_policy
ON sm_policy.objid = sm.snapMirrorPolicyId
LEFT JOIN
netapp_model.job_schedule AS job_sched
ON sm.jobScheduleId = job_sched.objid
WHERE
sm.objState = 'LIVE';
"""
networklifs_header = ['cluster', 'svm', 'lif_current_node', 'current_port',
'lif_home_node', 'home_port', 'lif_name',
'ip_address', 'netmask', 'netmask_length',
'mac_address', 'lif_type', 'lif_protocols',
'port_type', 'vlan_id', 'port_speed_Mb',
'dns_domain', 'mtu', 'lif_op_status',
'lif_adm_status', 'lif_role', 'port_role',
'port_status', 'svm_type', 'reportdate']
networklifs_sql = """\
SELECT
cluster.name AS cluster,
IF(vserver.name != 'Cluster',
vserver.name,
'') AS svm,
(SELECT
name
FROM
netapp_model.node
WHERE
node.objid = lif.currentNodeId) AS lif_current_node,
network_port.name AS current_port,
(SELECT
name
FROM
netapp_model.node
WHERE
node.objid = lif.homeNodeId) AS lif_home_node,
(SELECT
name
FROM
netapp_model.network_port
WHERE
network_port.objid = net_lif.homePortId) AS home_port,
lif.name AS lif_name,
net_lif.address AS ip_address,
net_lif.netmask AS netmask,
net_lif.netmaskLength AS netmask_length,
network_port.macAddress AS mac_address,
lif.lifType AS lif_type,
lif.dataProtocols AS lif_protocols,
network_port.portTypeRaw AS port_type,
network_port.vlanIdentifier AS vlan_id,
network_port.operationalSpeedRaw AS port_speed_Mb,
net_lif.dnsDomainName AS dns_domain,
network_port.mtu AS mtu,
lif.operationalStatusRaw AS lif_op_status,
lif.administrativeStatusRaw AS lif_adm_status,
lif.roleRaw AS lif_role,
network_port.roleRaw AS port_role,
network_port.linkStatusRaw AS port_status,
vserver.typeRaw AS svm_type,
CURDATE() AS reportdate
FROM
netapp_model.lif
JOIN
netapp_model.cluster ON cluster.objid = lif.clusterId
JOIN
netapp_model.vserver ON vserver.objid = lif.vserverId
JOIN
netapp_model.network_lif AS net_lif ON net_lif.uuid = lif.uuid
AND net_lif.objid = lif.objid
JOIN
netapp_model.network_port ON network_port.objid = net_lif.currentPortId
WHERE
lif.objState = 'LIVE';
"""
networkports_header = ['cluster', 'node', 'port', 'ip_space',
'broadcast_domain', 'mac_address', 'port_role',
'port_type', 'vlan_id', 'mtu', 'port_adm_speed_Mb',
'port_op_speed_Mb', 'port_health_status',
'port_health_degraded_reasons', 'reportdate']
networkports_sql = """\
SELECT
cluster.name AS cluster,
node.name AS node,
network_port.name AS port,
network_ip_space.name AS ip_space,
network_port_broadcast_domain.name AS broadcast_domain,
network_port.macAddress AS mac_address,
network_port.roleRaw AS port_role,
network_port.portTypeRaw AS port_type,
network_port.vlanIdentifier AS vlan_id,
network_port.mtu AS mtu,
network_port.administrativeSpeedRaw AS port_adm_speed_Mb,
network_port.operationalSpeedRaw AS port_op_speed_Mb,
network_port.healthStatusRaw AS port_health_status,
network_port.healthDegradedReasonsRaw AS port_health_degraded_reasons,
CURDATE() AS reportdate
FROM
netapp_model.network_port
JOIN
netapp_model.cluster ON cluster.objid = network_port.clusterId
JOIN
netapp_model.node ON node.objid = network_port.nodeId
JOIN
netapp_model.network_ip_space
ON network_ip_space.objid = network_port.networkIpSpaceId
LEFT JOIN
netapp_model.network_port_broadcast_domain
ON network_port_broadcast_domain.objid = network_port.networkPortBroadcastDomainId
WHERE
network_port.objState = 'LIVE';
"""
fcplifs_header = ['cluster', 'svm', 'lif_current_node',
'current_port_node', 'current_port',
'current_port_adapter', 'lif_home_node',
'home_port_node', 'home_port', 'home_port_adapter',
'lif_name', 'fabric_established', 'firmware_rev',
'lif_type', 'lif_protocols', 'port_type', 'port_speed',
'lif_op_status', 'lif_adm_status', 'lif_role',
'port_op_state', 'port_adm_status', 'svm_type',
'reportdate']
fcplifs_sql = """\
SELECT
cluster.name AS cluster,
IF(vserver.name != 'Cluster',
vserver.name,
'') AS svm,
(SELECT
name
FROM
netapp_model.node
WHERE
node.objid = lif.currentNodeId) AS lif_current_node,
fcp_port.nodeName AS current_port_node,
fcp_port.portName AS current_port,
fcp_port.adapter AS current_port_adapter,
(SELECT
name
FROM
netapp_model.node
WHERE
node.objid = lif.homeNodeId) AS lif_home_node,
(SELECT
fcp_port_alias.nodeName
FROM
fcp_port AS fcp_port_alias
WHERE
fcp_port_alias.objid = fcp_lif.homePortId) AS home_port_node,
(SELECT
fcp_port_alias.portName
FROM
fcp_port AS fcp_port_alias
WHERE
fcp_port_alias.objid = fcp_lif.homePortId) AS home_port,
(SELECT
fcp_port_alias.adapter
FROM
fcp_port AS fcp_port_alias
WHERE
fcp_port_alias.objid = fcp_lif.homePortId) AS home_port_adapter,
lif.name AS lif_name,
fcp_port.fabricEstablished AS fabric_established,
fcp_port.firmwareRev AS firmware_rev,
lif.lifType AS lif_type,
lif.dataProtocols AS lif_protocols,
fcp_port.PhysicalProtocolRaw AS port_type,
fcp_port.SpeedRaw AS port_speed,
lif.operationalStatusRaw AS lif_op_status,
lif.administrativeStatusRaw AS lif_adm_status,
lif.roleRaw AS lif_role,
fcp_port.StateRaw AS port_op_state,
fcp_port.StatusRaw AS port_adm_status,
vserver.typeRaw AS svm_type,
CURDATE() AS reportdate
FROM
netapp_model.lif
JOIN
netapp_model.cluster ON cluster.objid = lif.clusterId
JOIN
netapp_model.vserver ON vserver.objid = lif.vserverId
JOIN
netapp_model.fcp_lif ON fcp_lif.uuid = lif.uuid
AND fcp_lif.objid = lif.objid
JOIN
netapp_model.fcp_port ON fcp_port.objid = fcp_lif.currentPortId
WHERE
lif.objState = 'LIVE';
"""
fcpports_header = ['cluster', 'node', 'port_node_name', 'port',
'adapter', 'fabric_established', 'firmware_rev',
'adapter_info', 'max_speed_Gbit_sec', 'port_speed_Mbit',
'port_protocol', 'switch_port', 'port_status',
'adapter_state', 'adapter_media', 'reportdate']
fcpports_sql = """\
SELECT
cluster.name AS cluster,
node.name AS node,
fcp_port.nodeName AS port_node_name,
fcp_port.portName AS port,
fcp_port.adapter AS adapter,
fcp_port.fabricEstablished AS fabric_established,
fcp_port.firmwareRev AS firmware_rev,
fcp_port.infoName AS adapter_info,
fcp_port.maxSpeed AS max_speed_Gbit_sec,
fcp_port.SpeedRaw AS port_speed_Mbit,
fcp_port.PhysicalProtocolRaw AS port_protocol,
fcp_port.switchPort AS switch_port,
fcp_port.StatusRaw AS port_status,
fcp_port.StateRaw AS adapter_state,
fcp_port.MediaTypeRaw AS adapter_media,
CURDATE() AS reportdate
FROM
netapp_model.fcp_port
JOIN
netapp_model.cluster ON cluster.objid = fcp_port.clusterId
JOIN
netapp_model.node ON node.objid = fcp_port.nodeId
WHERE
fcp_port.objState = 'LIVE';
"""
reports = {
'disks': {
'csv_header': disks_header,
'query': disks_sql},
'aggrcapacity': {
'csv_header': aggrcapacity_header,
'query': aggrcapacity_sql},
'storagesystems': {
'csv_header': storagesystems_header,
'query': storagesystems_sql},
'svm': {
'csv_header': svm_header,
'query': svm_sql},
'volcapacity': {
'csv_header': volcapacity_header,
'query': volcapacity_sql},
'qtreecapacity': {
'csv_header': qtreecapacity_header,
'query': qtreecapacity_sql},
'nfsshares': {
'csv_header': nfsshares_header,
'query': nfsshares_sql},
'cifsshares': {
'csv_header': cifsshares_header,
'query': cifsshares_sql},
'luninfo': {
'csv_header': luninfo_header,
'query': luninfo_sql},
'storageefficiency': {
'csv_header': storagefficiency_header,
'query': storagefficiency_sql},
'snapshotpolicies': {
'csv_header': snapshotpolicies_header,
'query': snapshotpolicies_sql},
'replicationinfo': {
'csv_header': replicationinfo_header,
'query': replicationinfo_sql},
'networklifs': {
'csv_header': networklifs_header,
'query': networklifs_sql},
'networkports': {
'csv_header': networkports_header,
'query': networkports_sql},
'fcplifs': {
'csv_header': fcplifs_header,
'query': fcplifs_sql},
'fcpports': {
'csv_header': fcpports_header,
'query': fcpports_sql},
}
current_date = date.today()
country_code = conf['collector_data']['country']
conf['OCUM_database']['user'] = (
base64.b64decode(conf['OCUM_database']['user']))
conf['OCUM_database']['password'] = (
base64.b64decode(conf['OCUM_database']['password']))
data_dir = script_dir
if ('export_dir' in conf['collector_data']):
data_dir = conf['collector_data']['export_dir']
if not os.path.exists(data_dir):
try:
logger.debug('Creating missing data_dir path: %s'
% (data_dir))
os.makedirs(data_dir)
except OSError as e:
if e.errno != errno.EEXIST:
logger.error('Unabled to create data directory: %s'
% (os.path.basename(data_dir)))
logger.error('Export directory full path: %s'
% (data_dir))
logger.error('Please check if you have the right '
'permissions to create it')
sys.exit(13)
if not os.path.isdir(data_dir):
logger.error('Invalid export directory path provided: %s'
% (data_dir))
logger.error('Please provide a valid export directory path')
sys.exit(13)
if not os.access(data_dir, os.W_OK):
logger.error('No write permissions for export dir: %s' % (data_dir))
logger.error('Please add write permissions for the user you are '
'using to run the solution')
logger.error(' or ')
logger.error('Run the solution using another user with write '
'permissions on the provided export directory: %s'
% (data_dir))
sys.exit(13)
logger.debug('Setting OCUM MySQL source database name to: netapp_model')
conf['OCUM_database']['database'] = 'netapp_model'
logger.debug("Creating the MySQL connection")
try:
conn = mysql.connector.Connect(**conf['OCUM_database'])
except mysql.connector.Error as e:
logger.error('Unable to connect to OCUM MySQL server')
logger.error('Message: %s' % (e))
email_subject = ('[RO-MDR][NA-CDOT-COLLECTOR][%s][%s] - Unable to '
'connect to MySQL database'
% (country_code, hostname))
email_msg = 'ERROR: Unable to connect to MySQL Server
'
email_msg += ('Error message: %s' % (e))
to = conf['email']['to'].split(',')
send_mail(conf['email']['from'], to, email_subject, email_msg,
body_type='html', server=conf['email']['server'])
sys.exit(111)
cursor = conn.cursor(dictionary=True)
# List of the report file paths
report_files = list()
# List with the reports for which a report file has not been generated
missing_report_files = list()
logger.debug("Starting getting report data from OCUM database")
for report, report_data in reports.iteritems():
status, csv_file_path = build_report(report, cursor,
report_data['query'],
report_data['csv_header'],
data_dir, country_code)
if status:
logger.debug("%s: CSV file created" % (report))
report_files.append(csv_file_path)
else:
logger.debug("%s: Failed to create CSV files" % (report))
missing_report_files.append(report)
logger.debug('Closing MySQL connection')
conn.close()
logger.debug("Finished saving report data to CSV files")
logger.debug("Archiving CSV report files")
archive_file_name = ('%s_%s_na-cdot-mdr-data_%s'
% (country_code, hostname,
current_date.strftime('%Y%m%d'))
)
archive_path = archive_files(report_files, archive_file_name, data_dir)
ret_code = 0
if ('scp_transfer' in conf and
conf['scp_transfer']['enable'].lower() == 'true'):
logger.debug("Sending report archive to central CMDB server")
ret_code = scp_files([archive_path], conf['scp_transfer']['host'],
conf['scp_transfer']['user'],
conf['scp_transfer']['data_dir'],
conf['scp_transfer']['ssh_key'])
if ('email_transfer' in conf and
conf['email_transfer']['enable'].lower() == 'true'):
logger.debug('Sending report archive to central CMDB server')
email_subject = '[RO-MDR][NA-CDOT]'
email_msg = "PFA the report archive for: %s - %s" % (country_code,
hostname)
to = conf['email_transfer']['to'].split(',')
send_mail(conf['email_transfer']['from'], to, email_subject,
email_msg, files=[archive_path],
server=conf['email_transfer']['server'])
if ret_code != 0:
email_subject = ('[RO-MDR][NA-CDOT-COLLECTOR][%s][%s]: Unable to scp'
% (country_code, hostname))
email_msg = "Unable to send reports archive to central CMDB
"
email_msg += "OCUM Host: %s
" % (hostname)
email_msg += "Please investigate."
to = conf['email']['to'].split(',')
send_mail(conf['email']['from'], to, email_subject, email_msg,
body_type='html', server=conf['email']['server'])
logger.debug("Unable to send reports archive to central CMDB server")
sys.exit(22)
ocum_archives_glob = os.path.join(data_dir, '*_%s_*.zip' % (hostname))
ocum_archives = glob.glob(ocum_archives_glob)
if len(ocum_archives) == 3:
oldest_ocum_archive = sorted(ocum_archives)[0]
logger.debug('Deleting the OCUM report archive: %s'
% (oldest_ocum_archive))
os.remove(oldest_ocum_archive)
logger.debug('Deleted the OCUM report archive: %s'
% (oldest_ocum_archive))
logger.info('Success')
if __name__ == '__main__':
if len(sys.argv) > 2:
print "Too many arguments provided"
print "Run: python %s -help|-h for usage examples" % (__file__)
sys.exit(1)
logger.debug("")
logger.debug("Started %s " % script_name)
config_fname = '%s.ini' % (script_name)
config_file_path = os.path.join(script_dir, config_fname)
conf = load_config(config_file_path)
try:
if sys.argv[1] == '-check-config':
conf = load_config(config_file_path)
valid_config = check_config(conf, standalone=True)
if not valid_config:
logger.error('Invalid config file.')
sys.exit(2)
logger.info('Config file successfully validated')
elif sys.argv[1] == '-help' or sys.argv[1] == '-h':
usage_msg = ("Usage: python %s [-help|-h] | [check-config]" %
(__file__))
print usage_msg
print "Description:"
print "\tThe RO Cloud Netapp CDOT MDR data collecting service\n"
print "\tThis script is used to query the OCUM MySQL db and"\
"aggregate the data as required by the central MRD server.\n"
print "Arguments description:"
print "===All the supported arguments are optional===\n"
print "\t-help|-h: Prints this help message\n"
print "\t-check-config: Validates the config files"
print "Run examples:\n"
print "\t%s -help -> Displays the help message\n" % (__file__)
print "\t%s -check-config -> Validates the config\n" % (__file__)
print "\t%s -> Executes the data collection\n" % (__file__)
print "For more details please contact the Atos Romania Cloud " \
"Development Team "
else:
print "Uknown argument"
print "Run: python %s -help|-h for usage examples" % (__file__)
except IndexError:
valid_config = check_config(conf)
if not valid_config:
logger.error('Invalid config file')
sys.exit(3)
conf['OCUM_database']['ssl_disabled'] = "True"
conf['OCUM_database']['connection_timeout'] = 30
cdot_collect_db_data(conf)