Store time-series data into Mongo
This commit is contained in:
parent
9a605c2a68
commit
2c4e5167ee
|
@ -53,12 +53,9 @@ def main():
|
||||||
tag = utils.random_string()
|
tag = utils.random_string()
|
||||||
LOG.info('Using auto-generated tag "%s"', tag)
|
LOG.info('Using auto-generated tag "%s"', tag)
|
||||||
|
|
||||||
records = player.play_scenario(scenario, tag)
|
records, series = player.play_scenario(scenario, tag)
|
||||||
|
|
||||||
if records:
|
storage.store_data(cfg.CONF.mongo_url, cfg.CONF.mongo_db, records, series)
|
||||||
storage.store_data(records, cfg.CONF.mongo_url, cfg.CONF.mongo_db)
|
|
||||||
else:
|
|
||||||
LOG.warning('Execution generated no records')
|
|
||||||
|
|
||||||
report.generate_report(scenario, base_dir, cfg.CONF.mongo_url,
|
report.generate_report(scenario, base_dir, cfg.CONF.mongo_url,
|
||||||
cfg.CONF.mongo_db, cfg.CONF.book, tag)
|
cfg.CONF.mongo_db, cfg.CONF.book, tag)
|
||||||
|
|
|
@ -48,6 +48,7 @@ def play_setup(setup):
|
||||||
|
|
||||||
def play_execution(execution_playbook):
|
def play_execution(execution_playbook):
|
||||||
records = []
|
records = []
|
||||||
|
series = []
|
||||||
|
|
||||||
for play in execution_playbook:
|
for play in execution_playbook:
|
||||||
matrix = play.get('matrix')
|
matrix = play.get('matrix')
|
||||||
|
@ -62,35 +63,37 @@ def play_execution(execution_playbook):
|
||||||
|
|
||||||
for command_result in command_results:
|
for command_result in command_results:
|
||||||
if command_result.get('status') == 'OK':
|
if command_result.get('status') == 'OK':
|
||||||
record = dict(id=utils.make_id(),
|
payload = command_result['payload']
|
||||||
|
|
||||||
|
common = dict(id=utils.make_id(),
|
||||||
host=command_result['host'],
|
host=command_result['host'],
|
||||||
status=command_result['status'],
|
status=command_result['status'],
|
||||||
task=command_result['task'])
|
task=command_result['task'])
|
||||||
payload = command_result['payload']
|
common.update(payload['invocation']['module_args'])
|
||||||
record.update(payload['invocation']['module_args'])
|
|
||||||
record.update(payload)
|
|
||||||
|
|
||||||
# keep flat values only
|
if 'records' in payload:
|
||||||
for k, v in record.items():
|
for rec in payload['records']:
|
||||||
if isinstance(v, list) or isinstance(v, dict):
|
rec.update(common)
|
||||||
del record[k]
|
records.append(rec)
|
||||||
|
LOG.debug('New record: %s', rec)
|
||||||
|
|
||||||
if 'stdout' in record:
|
if 'series' in payload:
|
||||||
del record['stdout']
|
for rec in payload['series']:
|
||||||
|
rec.update(common)
|
||||||
|
series.append(rec)
|
||||||
|
LOG.debug('New time series: %s', rec)
|
||||||
|
|
||||||
LOG.debug('Record: %s', record)
|
return records, series
|
||||||
records.append(record)
|
|
||||||
|
|
||||||
return records
|
|
||||||
|
|
||||||
|
|
||||||
def tag_records(records, tag):
|
def add_tag(records, tag):
|
||||||
for r in records:
|
for r in records:
|
||||||
r['tag'] = tag
|
r['tag'] = tag
|
||||||
|
|
||||||
|
|
||||||
def play_scenario(scenario, tag):
|
def play_scenario(scenario, tag):
|
||||||
records = {}
|
records = []
|
||||||
|
series = []
|
||||||
|
|
||||||
if 'setup' in scenario:
|
if 'setup' in scenario:
|
||||||
play_setup(scenario['setup'])
|
play_setup(scenario['setup'])
|
||||||
|
@ -98,7 +101,8 @@ def play_scenario(scenario, tag):
|
||||||
if 'execution' in scenario:
|
if 'execution' in scenario:
|
||||||
execution = scenario['execution']
|
execution = scenario['execution']
|
||||||
|
|
||||||
records = play_execution(execution)
|
records, series = play_execution(execution)
|
||||||
tag_records(records, tag)
|
add_tag(records, tag)
|
||||||
|
add_tag(series, tag)
|
||||||
|
|
||||||
return records
|
return records, series
|
||||||
|
|
|
@ -21,12 +21,17 @@ from performa.engine import utils
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def store_data(records, mongo_url, mongo_db):
|
def store_data(mongo_url, mongo_db, records, series):
|
||||||
LOG.info('Store data to Mongo: %s', mongo_url)
|
LOG.info('Store data to Mongo: %s', mongo_url)
|
||||||
|
|
||||||
connection_params = utils.parse_url(mongo_url)
|
connection_params = utils.parse_url(mongo_url)
|
||||||
mongo_client = pymongo.MongoClient(**connection_params)
|
mongo_client = pymongo.MongoClient(**connection_params)
|
||||||
db = mongo_client.get_database(mongo_db)
|
db = mongo_client.get_database(mongo_db)
|
||||||
|
|
||||||
records_collection = db.get_collection('records')
|
if records:
|
||||||
records_collection.insert_many(records)
|
records_collection = db.get_collection('records')
|
||||||
|
records_collection.insert_many(records)
|
||||||
|
|
||||||
|
if series:
|
||||||
|
series_collection = db.get_collection('series')
|
||||||
|
series_collection.insert_many(series)
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/python
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
|
import tempfile
|
||||||
|
|
||||||
ATOP_FILE_NAME = '/tmp/performa.atop'
|
ATOP_FILE_NAME = os.path.join(tempfile.gettempdir(), 'performa.atop')
|
||||||
UNIQUE_NAME = 'performa_atop'
|
UNIQUE_NAME = 'performa_atop'
|
||||||
|
|
||||||
PREFIX_PATTERN = (
|
PREFIX_PATTERN = (
|
||||||
|
|
|
@ -68,17 +68,19 @@ def main():
|
||||||
'run'
|
'run'
|
||||||
) % module.params
|
) % module.params
|
||||||
|
|
||||||
|
start = int(time.time())
|
||||||
rc, stdout, stderr = module.run_command(cmd)
|
rc, stdout, stderr = module.run_command(cmd)
|
||||||
|
end = int(time.time())
|
||||||
result = dict(changed=True, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result.update(parse_sysbench_oltp(stdout))
|
parsed = parse_sysbench_oltp(stdout)
|
||||||
|
parsed['start'] = start
|
||||||
|
parsed['end'] = end
|
||||||
|
|
||||||
|
result = dict(records=[parsed])
|
||||||
module.exit_json(**result)
|
module.exit_json(**result)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result['exception'] = e
|
module.fail_json(msg=e, rc=rc, stderr=stderr, stdout=stdout)
|
||||||
|
|
||||||
module.fail_json(**result)
|
|
||||||
|
|
||||||
|
|
||||||
from ansible.module_utils.basic import * # noqa
|
from ansible.module_utils.basic import * # noqa
|
||||||
|
|
Loading…
Reference in New Issue