x
/
browbeat
Code Issues Proposed changes

Query Elastic to compare software-metadata 

Method to query ElasticSearch for a specific set of browbeat_uuids and
compare the metadata to determine if there are differences.

This work will also tell the user if a option or value is missing.

Eventually, I would like to see us query Elastic for collectd data to
see if there has been CPU/Memory/DiskIO increases during a specific
Browbeat run -- this is a longer-term goal.

Example of this :
https://gist.github.com/jtaleric/ffc1508eba3cba9515ca24cfcf23583c

Change-Id: Ie65e2c3d505aa2f19ba10109276ba982ee4ab67b
This commit is contained in:
Joe Talerico 2017-05-24 14:12:41 -04:00
parent 6f9aef7987
commit 21900bffad
3 changed files with 230 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
browbeat.py
View File

@ -11,24 +11,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from lib.Elastic import browbeat_uuid
import argparse
import datetime
import lib.Elastic
import lib.PerfKit
import lib.Rally
import lib.Shaker
import lib.Yoda
import lib.WorkloadBase
import lib.Tools
import argparse
import logging
import os
import sys
import time
import datetime
import os
_workload_opts = ['perfkit', 'rally', 'shaker', 'yoda']
_config_file = 'browbeat-config.yaml'
debug_log_file = 'log/debug.log'
def main():
tools = lib.Tools.Tools()
parser = argparse.ArgumentParser(
@ -41,14 +42,21 @@ def main():
help='Provide Browbeat YAML configuration file. Default is ./{}'.format(_config_file))
parser.add_argument('workloads', nargs='*', help='Browbeat workload(s). Takes a space separated'
' list of workloads ({}) or \"all\"'.format(', '.join(_workload_opts)))
parser.add_argument('--debug', action='store_true', help='Enable Debug messages')
parser.add_argument('-p','--postprocess',
dest="path",help="Path to process, ie results/20170101/")
parser.add_argument('--debug', action='store_true',
help='Enable Debug messages')
parser.add_argument('-p', '--postprocess',
dest="path", help="Path to process, ie results/20170101/")
parser.add_argument('-c', '--compare',
help="Compare metadata", dest="compare",
choices=['software-metadata'])
parser.add_argument('-u', '--uuid',
help="UUIDs to pass", dest="uuids", nargs=2)
_cli_args = parser.parse_args()
_logger = logging.getLogger('browbeat')
_logger.setLevel(logging.DEBUG)
_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)7s - %(message)s')
_formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)7s - %(message)s')
_formatter.converter = time.gmtime
_dbg_file = logging.FileHandler(debug_log_file)
_dbg_file.setLevel(logging.DEBUG)
@ -67,24 +75,35 @@ def main():
# Load Browbeat yaml config file:
_config = tools._load_config(_cli_args.setup)
if _cli_args.compare == "software-metadata":
es = lib.Elastic.Elastic(_config, "BrowbeatCLI")
es.compare_metadata("_all", 'controller', _cli_args.uuids)
exit(0)
if _cli_args.compare:
parser.print_help()
exit(1)
# Default to all workloads
if _cli_args.workloads == []:
_cli_args.workloads.append('all')
if _cli_args.path :
if _cli_args.path:
return tools.post_process(_cli_args)
if len(_cli_args.workloads) == 1 and 'all' in _cli_args.workloads:
_cli_args.workloads = _workload_opts
invalid_wkld = [wkld for wkld in _cli_args.workloads if wkld not in _workload_opts]
invalid_wkld = [
wkld for wkld in _cli_args.workloads if wkld not in _workload_opts]
if invalid_wkld:
_logger.error("Invalid workload(s) specified: {}".format(invalid_wkld))
if 'all' in _cli_args.workloads:
_logger.error("If you meant 'all' use: './browbeat.py all' or './browbeat.py'")
_logger.error(
"If you meant 'all' use: './browbeat.py all' or './browbeat.py'")
exit(1)
else:
time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
_logger.info("Browbeat test suite kicked off")
_logger.info("Browbeat UUID: {}".format(browbeat_uuid))
_logger.info("Browbeat UUID: {}".format(lib.Elastic.browbeat_uuid))
if _config['elasticsearch']['enabled']:
_logger.info("Checking for Metadata")
metadata_exists = tools.check_metadata()
@ -93,11 +112,12 @@ def main():
" metadata files do not exist")
_logger.info("Gathering Metadata")
tools.gather_metadata()
elif _config['elasticsearch']['regather'] :
elif _config['elasticsearch']['regather']:
_logger.info("Regathering Metadata")
tools.gather_metadata()
_logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads)))
_logger.info("Running workload(s): {}".format(
','.join(_cli_args.workloads)))
for wkld_provider in _cli_args.workloads:
if wkld_provider in _config:
if _config[wkld_provider]['enabled']:
@ -106,11 +126,12 @@ def main():
_logger.warning("{} is not enabled in {}".format(wkld_provider,
_cli_args.setup))
else:
_logger.error("{} is missing in {}".format(wkld_provider, _cli_args.setup))
_logger.error("{} is missing in {}".format(
wkld_provider, _cli_args.setup))
result_dir = _config['browbeat']['results']
lib.WorkloadBase.WorkloadBase.print_report(result_dir, time_stamp)
_logger.info("Saved browbeat result summary to {}".format(
os.path.join(result_dir,time_stamp + '.' + 'report')))
os.path.join(result_dir, time_stamp + '.' + 'report')))
lib.WorkloadBase.WorkloadBase.print_summary()
browbeat_rc = 0
@ -120,15 +141,17 @@ def main():
browbeat_rc = 2
if browbeat_rc == 1:
_logger.info("Browbeat finished with test failures, UUID: {}".format(browbeat_uuid))
sys.exit(browbeat_rc)
_logger.info("Browbeat finished with test failures, UUID: {}".format(
lib.Elastic.browbeat_uuid))
sys.exit(browbeat_rc)
elif browbeat_rc == 2:
_logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
.format(browbeat_uuid))
sys.exit(browbeat_rc)
_logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
.format(lib.Elastic.browbeat_uuid))
sys.exit(browbeat_rc)
else:
_logger.info("Browbeat finished successfully, UUID: {}".format(browbeat_uuid))
sys.exit(0)
_logger.info("Browbeat finished successfully, UUID: {}".format(
lib.Elastic.browbeat_uuid))
sys.exit(0)
if __name__ == '__main__':
sys.exit(main())

38
doc/source/usage.rst
View File

@ -38,7 +38,7 @@ Run performance stress tests through Browbeat
(browbeat-venv)[stack@ospd browbeat]$ ./browbeat.py <workload> #perfkit, rally, shaker or "all"
Running PerfKitBenchmarker
==========================
---------------------------
Work is on-going to utilize PerfKitBenchmarker as a workload provider to
Browbeat. Many benchmarks work out of the box with Browbeat. You must
@ -69,7 +69,8 @@ browbeat-config.yaml:
(browbeat-venv)[stack@ospd browbeat]$ ./browbeat.py perfkit -s browbeat-config.yaml
Running Shaker
==============
---------------
Running Shaker requires the shaker image to be built, which in turn requires
instances to be able to access the internet. The playbooks for this installation
have been described in the installation documentation but for the sake of
@ -215,7 +216,7 @@ I would suggest bulk introspection for testing documented TripleO workflows and
individual introspection to test the performance of introspection itself.
Interpreting Browbeat Results
=============================
------------------------------
By default results for each test will be placed in a timestamped folder `results/` inside your Browbeat folder.
Each run folder will contain output files from the various workloads and benchmarks that ran during that Browbeat
@ -232,7 +233,7 @@ and Kibana to view them more easily.
Working with Multiple Clouds
============================
-----------------------------
If you are running playbooks from your local machine you can run against more
than one cloud at the same time. To do this, you should create a directory
@ -249,3 +250,32 @@ per-cloud and clone Browbeat into that specific directory:
[browbeat@laptop ansible]$ ssh -F ssh-config overcloud-controller-0 # Takes you to first controller
Repeat the above steps for as many clouds as you have to run playbooks against your clouds.
Compare software-metadata from two different runs
--------------------------------------------------
Browbeat's metadata is great to help build visuals in Kibana by querying on specific metadata fields, but sometimes
we need to see what the difference between two builds might be. Kibana doesn't have a good way to show this, so we
added an option to Browbeat CLI to query ElasticSearch.
To use :
::
$ python browbeat.py --compare software-metadata --uuid "browbeat-uuid-1" "browbeat-uuid-2"
Real world use-case, we had two builds in our CI that used the exact same DLRN hash, however the later build had a
10x performance hit for two Neutron operations, router-create and add-interface-to-router. Given we had exactly the
same DLRN hash, the only difference could be how things were configured. Using this new code, we could quickly identify
the difference -- TripleO enabled l3_ha.
::
[rocketship:browbeat] jtaleric:browbeat$ python browbeat.py --compare software-metadata --uuid "3fc2f149-7091-4e16-855a-60738849af17" "6738eed7-c8dd-4747-abde-47c996975a57"
2017-05-25 02:34:47,230 - browbeat.Tools - INFO - Validating the configuration file passed by the user
2017-05-25 02:34:47,311 - browbeat.Tools - INFO - Validation successful
2017-05-25 02:34:47,311 - browbeat.Elastic - INFO - Querying Elastic : index [_all] : role [controller] : uuid [3fc2f149-7091-4e16-855a-60738849af17]
2017-05-25 02:34:55,684 - browbeat.Elastic - INFO - Querying Elastic : index [_all] : role [controller] : uuid [6738eed7-c8dd-4747-abde-47c996975a57]
2017-05-25 02:35:01,165 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-2] Service [neutron] l3_ha [False]
2017-05-25 02:35:01,168 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-1] Service [neutron] l3_ha [False]
2017-05-25 02:35:01,172 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-0] Service [neutron] l3_ha [False]

170
lib/Elastic.py
View File

@ -18,6 +18,7 @@ import uuid
import sys
import time
import os
import re
browbeat_uuid = uuid.uuid4()
@ -34,7 +35,8 @@ class Elastic(object):
)
self.workload = workload
today = datetime.datetime.today()
self.index = "{}-{}-{}".format(tool, workload, today.strftime('%Y.%m.%d'))
self.index = "{}-{}-{}".format(tool,
workload, today.strftime('%Y.%m.%d'))
def load_json(self, result):
json_data = None
@ -63,15 +65,23 @@ class Elastic(object):
result[_meta['name']] = json.load(jdata)
except Exception:
self.logger.error(
"Error loading Metadata file : {}".format(_meta['file']))
self.logger.error("Please make sure the metadata file exists and"
" is valid JSON or run the playbook ansible/gather/site.yml"
" before running the Browbeat test Suite")
"Error loading Metadata file : {}".format(
_meta['file']))
self.logger.error(
"Please make sure the metadata file exists and"
" is valid JSON or run the playbook ansible/gather/site.yml"
" before running the Browbeat test Suite")
sys.exit(1)
return result
def index_result(self, result, test_name, result_dir, identifier='', _type='result',
_id=None):
def index_result(
self,
result,
test_name,
result_dir,
identifier='',
_type='result',
_id=None):
retry = 2
result['browbeat_uuid'] = str(browbeat_uuid)
result['cloud_name'] = self.config['browbeat']['cloud_name']
@ -88,18 +98,138 @@ class Elastic(object):
format(self.index, result['browbeat_uuid']))
return True
except Exception as Err:
self.logger.error("Error pushing data to Elasticsearch, going to retry"
" in 10 seconds")
self.logger.error(
"Error pushing data to Elasticsearch, going to retry"
" in 10 seconds")
self.logger.error("Exception: {}".format(Err))
time.sleep(10)
if i == (retry-1):
self.logger.error("Pushing Data to Elasticsearch failed in spite of retry,"
" dumping JSON")
elastic_file = os.path.join(result_dir,
test_name + '-' + identifier + '-elastic' +
'.' + 'json')
with open(elastic_file, 'w') as result_file:
json.dump(result, result_file, indent=4, sort_keys=True)
self.logger.info("Saved Elasticsearch consumable result JSON to {}".
format(elastic_file))
return False
if i == (retry - 1):
self.logger.error(
"Pushing Data to Elasticsearch failed in spite of retry,"
" dumping JSON")
elastic_file = os.path.join(
result_dir, test_name + '-' + identifier + '-elastic' + '.' + 'json')
with open(elastic_file, 'w') as result_file:
json.dump(result, result_file,
indent=4, sort_keys=True)
self.logger.info(
"Saved Elasticsearch consumable result JSON to {}".format(elastic_file))
return False
def get_software_metadata(self, index, role, browbeat_uuid):
nodes = {}
results = self.query_uuid(index, browbeat_uuid)
pattern = re.compile(".*{}.*".format(role))
if results:
for result in results:
for metadata in result['_source']['software-metadata']:
for service in metadata:
if pattern.match(metadata[service]['node_name']):
if metadata[service]['node_name'] not in nodes:
nodes[metadata[service][
'node_name']] = metadata
return nodes
else:
self.logger.error("UUID {} wasn't found".format(browbeat_uuid))
return False
"""
Currently this function will only compare two uuids. I (rook) am not convinced it is worth
the effort to engineer anything > 2.
"""
def compare_metadata(self, index, role, uuids):
meta = []
for browbeat_uuid in uuids:
self.logger.info(
"Querying Elastic : index [{}] : role [{}] : browbeat_uuid [{}] ".format(
index, role, browbeat_uuid))
software_metadata = self.get_software_metadata(index, role, browbeat_uuid)
if software_metadata:
meta.append(software_metadata)
else:
return False
ignore = [
"connection",
"admin_url",
"bind_host",
"rabbit_hosts",
"auth_url",
"public_bind_host",
"host",
"key",
"url",
"auth_uri",
"coordination_url",
"swift_authurl",
"admin_token",
"memcached_servers",
"api_servers",
"osapi_volume_listen",
"nova_url",
"coordination_url",
"memcache_servers",
"novncproxy_host",
"backend_url",
"novncproxy_base_url",
"metadata_listen",
"osapi_compute_listen",
"admin_bind_host",
"glance_api_servers",
"iscsi_ip_address",
"registry_host",
"auth_address",
"swift_key",
"auth_encryption_key",
"metadata_proxy_shared_secret",
"telemetry_secret",
"heat_metadata_server_url",
"heat_waitcondition_server_url",
"transport_url"]
if len(meta) < 2:
self.logger.error("Unable to compare data-sets")
return False
for host in meta[0]:
if host not in meta[1]:
self.logger.error("Deployment differs: "
"Host [{}] missing ".format(host))
continue
for service in meta[0][host]:
for options in meta[0][host][service].keys():
if options not in meta[1][host][service]:
self.logger.error(
"Missing Option : "
"Host [{}] Service [{}] {}".format(
host, service, options))
continue
if isinstance(meta[0][host][service][options], dict):
for key in meta[0][host][service][options].keys():
if key not in ignore:
if key in meta[1][host][service][options]:
value = meta[0][host][
service][options][key]
new_value = meta[1][host][
service][options][key]
if value != new_value:
self.logger.info(
"Difference found : "
"Host [{}] Service [{}] Section {} {} [{}]".format(
host,
service,
options,
key,
meta[0][host][service][options][key]))
else:
self.logger.info(
"Missing Value : "
"Host [{}] Service [{}] {} [{}]".format(
host, service, options, key))
def query_uuid(self, index, browbeat_uuid):
body = {'query': {"match": {"browbeat_uuid": {
"query": browbeat_uuid, "type": "phrase"}}}}
results = self.es.search(index=index, doc_type='result', body=body)
if len(results['hits']['hits']) > 0:
return results['hits']['hits']
else:
return False