Proof-of-concept

The PoC shows base feature of the toolkit:
 * execution plan is specified in scenario
 * scenario is transformed into ansible playbook which is executed
on remote hosts
 * the data is stored into mongo db
 * report is generated based on template, and it can contain charts
and table data processed by mongo aggregate pipeline
This commit is contained in:
Ilya Shakhat 2016-02-19 13:43:06 +03:00
parent 74de6be091
commit 3bd574c45b
23 changed files with 1201 additions and 59 deletions

View File

@ -38,7 +38,7 @@ master_doc = 'index'
# General information about the project.
project = u'performa'
copyright = u'2013, OpenStack Foundation'
copyright = u'2016, OpenStack Foundation'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True

View File

@ -4,7 +4,7 @@
contain the root `toctree` directive.
Welcome to performa's documentation!
========================================================
====================================
Contents:

View File

@ -1,19 +0,0 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pbr.version
__version__ = pbr.version.VersionInfo(
'performa').version_string()

View File

View File

@ -0,0 +1,139 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple
from ansible.executor import task_queue_manager
from ansible import inventory
from ansible.parsing import dataloader
from ansible.playbook import play
from ansible.plugins import callback
from ansible.vars import VariableManager
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class MyCallback(callback.CallbackBase):
CALLBACK_VERSION = 2.0
CALLBACK_TYPE = 'stdout'
CALLBACK_NAME = 'myown'
def __init__(self, storage, display=None):
super(MyCallback, self).__init__(display)
self.storage = storage
def _store(self, result, status):
record = dict(host=result._host.get_name(),
status=status,
task=result._task.get_name(),
payload=result._result)
self.storage.append(record)
def v2_runner_on_failed(self, result, ignore_errors=False):
super(MyCallback, self).v2_runner_on_failed(result)
self._store(result, 'FAILED')
def v2_runner_on_ok(self, result):
super(MyCallback, self).v2_runner_on_ok(result)
self._store(result, 'OK')
def v2_runner_on_skipped(self, result):
super(MyCallback, self).v2_runner_on_skipped(result)
self._store(result, 'SKIPPED')
def v2_runner_on_unreachable(self, result):
super(MyCallback, self).v2_runner_on_unreachable(result)
self._store(result, 'UNREACHABLE')
Options = namedtuple('Options',
['connection', 'password', 'module_path', 'forks',
'remote_user',
'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become',
'become_method', 'become_user', 'verbosity', 'check'])
def _run(play_source, host_list):
variable_manager = VariableManager()
loader = dataloader.DataLoader()
options = Options(connection='smart', password='swordfish',
module_path='/path/to/mymodules',
forks=100, remote_user='developer',
private_key_file=None,
ssh_common_args=None, ssh_extra_args=None,
sftp_extra_args=None, scp_extra_args=None, become=None,
become_method=None, become_user=None, verbosity=100,
check=False)
passwords = dict(vault_pass='secret')
# create inventory and pass to var manager
inventory_inst = inventory.Inventory(loader=loader,
variable_manager=variable_manager,
host_list=host_list)
variable_manager.set_inventory(inventory_inst)
# create play
play_inst = play.Play().load(play_source,
variable_manager=variable_manager,
loader=loader)
storage = []
callback = MyCallback(storage)
# actually run it
tqm = None
try:
tqm = task_queue_manager.TaskQueueManager(
inventory=inventory_inst,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=passwords,
stdout_callback=callback,
)
tqm.run(play_inst)
finally:
if tqm is not None:
tqm.cleanup()
return storage
def run_command(command, host_list):
hosts = ','.join(host_list) + ','
# tasks = [dict(action=dict(module='shell', args=command))]
tasks = [{'command': command}]
play_source = dict(
hosts=host_list,
gather_facts='no',
tasks=tasks,
)
return _run(play_source, hosts)
def run_playbook(playbook, host_list):
for play_source in playbook:
hosts = ','.join(host_list) + ','
play_source['hosts'] = hosts
play_source['gather_facts'] = 'no'
_run(play_source, hosts)

67
performa/engine/config.py Normal file
View File

@ -0,0 +1,67 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
from oslo_config import cfg
from oslo_config import types
from performa.engine import utils
SCENARIOS = 'performa/scenarios/'
class Endpoint(types.String):
def __call__(self, value):
value = str(value)
utils.parse_url(value)
return value
def __repr__(self):
return "Endpoint host[:port]"
MAIN_OPTS = [
cfg.StrOpt('scenario',
default=utils.env('PERFORMA_SCENARIO'),
required=True,
help=utils.make_help_options(
'Scenario to play. Can be a file name or one of aliases: '
'%s. Defaults to env[PERFORMA_SCENARIO].', SCENARIOS,
type_filter=lambda x: x.endswith('.yaml'))),
cfg.Opt('mongo-url',
default=utils.env('PERFORMA_MONGO_URL'),
required=True,
type=Endpoint(),
help='Mongo URL, defaults to env[PERFORMA_MONGO_URL].'),
cfg.StrOpt('mongo-db',
default=utils.env('PERFORMA_MONGO_DB'),
required=True,
help='Mongo DB, defaults to env[PERFORMA_MONGO_DB].'),
cfg.ListOpt('hosts',
default=utils.env('PERFORMA_HOSTS'),
required=True,
help='List of hosts, defaults to env[PERFORMA_MONGO_URL].'),
cfg.StrOpt('book',
default=utils.env('PERFORMA_BOOK'),
help='Generate report in ReST format and store it into the '
'specified folder, defaults to env[PERFORMA_BOOK]. '),
]
def list_opts():
yield (None, copy.deepcopy(MAIN_OPTS))

49
performa/engine/main.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from oslo_config import cfg
from oslo_log import log as logging
from performa.engine import config
from performa.engine import player
from performa.engine import report
from performa.engine import storage
from performa.engine import utils
LOG = logging.getLogger(__name__)
def main():
utils.init_config_and_logging(config.MAIN_OPTS)
scenario_file_path = utils.get_absolute_file_path(
cfg.CONF.scenario,
alias_mapper=lambda f: config.SCENARIOS + '%s.yaml' % f)
scenario = utils.read_yaml_file(scenario_file_path)
base_dir = os.path.dirname(scenario_file_path)
records = player.play_scenario(scenario)
storage.store_data(records, cfg.CONF.mongo_url, cfg.CONF.mongo_db)
report.generate_report(scenario, base_dir, cfg.CONF.mongo_url,
cfg.CONF.mongo_db, cfg.CONF.book)
if __name__ == "__main__":
main()

94
performa/engine/player.py Normal file
View File

@ -0,0 +1,94 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import re
from oslo_config import cfg
from oslo_log import log as logging
from performa.engine import ansible_runner
from performa.engine import utils
from performa import executors as executors_classes
LOG = logging.getLogger(__name__)
def run_command(command):
return ansible_runner.run_command(command, cfg.CONF.hosts)
def _make_test_title(test, params=None):
s = test.get('title') or test.get('class')
if params:
s += ' '.join([','.join(['%s=%s' % (k, v) for k, v in params.items()
if k != 'host'])])
return re.sub(r'[^\x20-\x7e\x80-\xff]+', '_', s)
def _pick_tests(tests, matrix):
matrix = matrix or {}
for test in tests:
for params in utils.algebraic_product(**matrix):
parametrized_test = copy.deepcopy(test)
parametrized_test.update(params)
parametrized_test['title'] = _make_test_title(test, params)
yield parametrized_test
def play_preparation(preparation):
ansible_playbook = preparation.get('ansible-playbook')
if ansible_playbook:
ansible_runner.run_playbook(ansible_playbook, cfg.CONF.hosts)
def play_execution(execution):
records = []
matrix = execution.get('matrix')
for test in _pick_tests(execution['tests'], matrix):
executor = executors_classes.get_executor(test)
command = executor.get_command()
command_results = run_command(command)
for command_result in command_results:
record = dict(id=utils.make_id(),
host=command_result['host'],
status=command_result['status'])
record.update(test)
if command_result.get('status') == 'OK':
er = executor.process_reply(command_result['payload'])
record.update(er)
records.append(record)
return records
def play_scenario(scenario):
records = {}
if 'preparation' in scenario:
play_preparation(scenario['preparation'])
if 'execution' in scenario:
execution = scenario['execution']
records = play_execution(execution)
return records

132
performa/engine/report.py Normal file
View File

@ -0,0 +1,132 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import errno
import functools
import os
import tempfile
import jinja2
from oslo_config import cfg
from oslo_log import log as logging
import pygal
from pygal import style
import pymongo
import yaml
from performa.engine import config
from performa.engine import utils
LOG = logging.getLogger(__name__)
def generate_chart(chart_str, records_collection, doc_folder):
chart = yaml.safe_load(chart_str)
pipeline = chart.get('pipeline')
title = chart.get('title')
axes = chart.get('axes') or dict(x='x', y='y')
chart_data = records_collection.aggregate(pipeline)
line = []
table = '''
.. list-table:: %(title)s
:header-rows: 1
* - %(x)s
- %(y)s
''' % dict(title=title, x=axes['x'], y=axes['y'])
for chart_rec in chart_data:
line.append((chart_rec['x'], chart_rec['y']))
table += (' *\n' +
'\n'.join(' - %d' % chart_rec[v] for v in ('x', 'y')) +
'\n')
xy_chart = pygal.XY(style=style.RedBlueStyle,
fill=True,
legend_at_bottom=True,
include_x_axis=True,
x_title=axes['x'])
xy_chart.add(axes['y'], line)
chart_filename = utils.strict(title)
abs_chart_filename = '%s.svg' % os.path.join(doc_folder, chart_filename)
xy_chart.render_to_file(abs_chart_filename)
doc = '.. image:: %s.*\n\n' % chart_filename
doc += table
return doc
def _make_dir(name):
try:
os.makedirs(name)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder):
LOG.info('Generate report')
doc_folder = doc_folder or tempfile.mkdtemp(prefix='performa')
connection_params = utils.parse_url(mongo_url)
mongo_client = pymongo.MongoClient(**connection_params)
db = mongo_client.get_database(db_name)
records_collection = db.get_collection('records')
report_definition = scenario['report']
report_template = report_definition['template']
_make_dir(doc_folder)
jinja_env = jinja2.Environment()
jinja_env.filters['chart'] = functools.partial(
generate_chart,
records_collection=records_collection,
doc_folder=doc_folder)
template = utils.read_file(report_template, base_dir=base_dir)
compiled_template = jinja_env.from_string(template)
rendered_template = compiled_template.render()
index = open(os.path.join(doc_folder, 'index.rst'), 'w+')
index.write(rendered_template)
index.close()
LOG.info('The report is written to %s', doc_folder)
def main():
utils.init_config_and_logging(config.MAIN_OPTS)
scenario_file_path = utils.get_absolute_file_path(
cfg.CONF.scenario,
alias_mapper=lambda f: config.SCENARIOS + '%s.yaml' % f)
scenario = utils.read_yaml_file(scenario_file_path)
base_dir = os.path.dirname(scenario_file_path)
generate_report(scenario, base_dir, cfg.CONF.mongo_url, cfg.CONF.mongo_db,
cfg.CONF.book)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,32 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
import pymongo
from performa.engine import utils
LOG = logging.getLogger(__name__)
def store_data(records, mongo_url, mongo_db):
LOG.info('Store data to Mongo: %s', mongo_url)
connection_params = utils.parse_url(mongo_url)
mongo_client = pymongo.MongoClient(**connection_params)
db = mongo_client.get_database(mongo_db)
records_collection = db.get_collection('records')
records_collection.insert_many(records)

216
performa/engine/utils.py Normal file
View File

@ -0,0 +1,216 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import itertools
import logging as std_logging
import os
import random
import uuid
from oslo_config import cfg
from oslo_log import log as logging
import re
import six
import yaml
LOG = logging.getLogger(__name__)
def env(*_vars, **kwargs):
"""Returns the first environment variable set.
If none are non-empty, defaults to '' or keyword arg default.
"""
for v in _vars:
value = os.environ.get(v)
if value:
return value
return kwargs.get('default', None)
def validate_required_opts(conf, opts):
# all config parameters default to ENV values, that's why standard
# check of required options doesn't work and needs to be done manually
for opt in opts:
if opt.required and not conf[opt.dest]:
raise cfg.RequiredOptError(opt.name)
def init_config_and_logging(opts):
conf = cfg.CONF
conf.register_cli_opts(opts)
conf.register_opts(opts)
logging.register_options(conf)
logging.set_defaults()
try:
conf(project='performa')
validate_required_opts(conf, opts)
except cfg.RequiredOptError as e:
print('Error: %s' % e)
conf.print_usage()
exit(1)
logging.setup(conf, 'performa')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
def resolve_relative_path(file_name):
path = os.path.normpath(os.path.join(
os.path.dirname(__import__('performa').__file__), '../', file_name))
if os.path.exists(path):
return path
def get_absolute_file_path(file_name, base_dir='', alias_mapper=None):
full_path = os.path.normpath(os.path.join(base_dir, file_name))
if alias_mapper: # interpret file_name as alias
alias_path = resolve_relative_path(alias_mapper(file_name))
if alias_path:
full_path = alias_path
LOG.info('Alias "%s" is resolved into file "%s"', file_name,
full_path)
if not os.path.exists(full_path):
# treat file_name as relative to act's package root
full_path = os.path.normpath(
os.path.join(os.path.dirname(__import__('performa').__file__),
'../', file_name))
if not os.path.exists(full_path):
msg = ('File %s not found by absolute nor by relative path' %
file_name)
raise IOError(msg)
return full_path
def read_file(file_name, base_dir='', alias_mapper=None):
full_path = get_absolute_file_path(file_name, base_dir, alias_mapper)
fd = None
try:
fd = open(full_path)
return fd.read()
except IOError as e:
LOG.error('Error reading file: %s', e)
raise
finally:
if fd:
fd.close()
def write_file(data, file_name, base_dir=''):
full_path = os.path.normpath(os.path.join(base_dir, file_name))
fd = None
try:
fd = open(full_path, 'w')
return fd.write(data)
except IOError as e:
LOG.error('Error writing file: %s', e)
raise
finally:
if fd:
fd.close()
def read_yaml_file(file_name, base_dir='', alias_mapper=None):
raw = read_file(file_name, base_dir, alias_mapper)
try:
parsed = yaml.safe_load(raw)
return parsed
except Exception as e:
LOG.error('Failed to parse file %(file)s in YAML format: %(err)s',
dict(file=file_name, err=e))
def read_uri(uri):
try:
req = six.moves.urllib.request.Request(url=uri)
fd = six.moves.urllib.request.urlopen(req)
raw = fd.read()
fd.close()
return raw
except Exception as e:
LOG.warn('Error "%(error)s" while reading uri %(uri)s',
{'error': e, 'uri': uri})
def random_string(length=6):
return ''.join(random.sample('adefikmoprstuz', length))
def make_id():
return str(uuid.uuid4())
def make_help_options(message, base, type_filter=None):
path = resolve_relative_path(base)
files = itertools.chain.from_iterable(
[map(functools.partial(os.path.join, root), files)
for root, dirs, files in os.walk(path)]) # list of files in a tree
if type_filter:
files = (f for f in files if type_filter(f)) # filtered list
rel_files = map(functools.partial(os.path.relpath, start=path), files)
return message % ', '.join('"%s"' % f.partition('.')[0]
for f in sorted(rel_files))
def algebraic_product(**kwargs):
position_to_key = {}
values = []
total = 1
for key, item in six.iteritems(kwargs):
position_to_key[len(values)] = key
if type(item) != list:
item = [item] # enclose single item into the list
values.append(item)
total *= len(item)
LOG.debug('Total number of permutations is: %s', total)
for chain in itertools.product(*values):
result = {}
for position, key in six.iteritems(position_to_key):
result[key] = chain[position]
yield result
def strict(s):
return re.sub(r'[^\w\d]+', '_', re.sub(r'\(.+\)', '', s)).lower()
def parse_url(url):
arr = url.split(':')
result = {}
if len(arr) > 0:
result['host'] = arr[0]
if len(arr) > 1:
if arr[1].isdigit():
result['port'] = int(arr[1])
else:
raise ValueError('URL should be in form of host[:port], '
'but got: %s', url)
else:
ValueError('URL should not be empty')
return result

View File

@ -0,0 +1,29 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from performa.executors import shell
from performa.executors import sysbench
EXECUTORS = {
'sysbench-oltp': sysbench.SysbenchOltpExecutor,
'_default': shell.ShellExecutor,
}
def get_executor(test_definition):
# returns executor of the specified test on the specified agent
executor_class = test_definition['class']
klazz = EXECUTORS.get(executor_class, EXECUTORS['_default'])
return klazz(test_definition)

View File

@ -0,0 +1,61 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class CommandLine(object):
def __init__(self, command):
self.tokens = [command]
def add(self, param_name, param_value=None):
token = param_name
if param_value is not None:
token += '=' + str(param_value)
self.tokens.append(token)
def make(self):
return ' '.join(self.tokens)
class BaseExecutor(object):
def __init__(self, test_definition):
super(BaseExecutor, self).__init__()
self.test_definition = test_definition
def get_expected_duration(self):
return self.test_definition.get('time') or 60
def get_command(self):
return None
def process_reply(self, message):
LOG.debug('Test %s finished with %s',
self.test_definition, message)
return dict(stdout=message.get('stdout'),
stderr=message.get('stderr'),
command=self.get_command())
def process_failure(self):
return dict(command=self.get_command())
class ExecutorException(Exception):
def __init__(self, record, message):
super(ExecutorException, self).__init__(message)
self.record = record

View File

@ -0,0 +1,30 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from performa.executors import base
LOG = logging.getLogger(__name__)
class ShellExecutor(base.BaseExecutor):
def get_command(self):
if 'program' in self.test_definition:
cmd = base.CommandLine(self.test_definition['program'])
elif 'script' in self.test_definition:
cmd = base.Script(self.test_definition['script'])
return cmd.make()

View File

@ -0,0 +1,91 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from oslo_log import log as logging
from performa.executors import base
LOG = logging.getLogger(__name__)
TEST_STATS = re.compile(
'\s+queries performed:\s*\n'
'\s+read:\s+(?P<queries_read>\d+)\s*\n'
'\s+write:\s+(?P<queries_write>\d+).*\n'
'\s+other:\s+(?P<queries_other>\d+).*\n'
'\s+total:\s+(?P<queries_total>\d+).*\n',
flags=re.MULTILINE | re.DOTALL
)
PATTERNS = [
r'sysbench (?P<version>[\d\.]+)',
TEST_STATS,
r'\s+transactions:\s+(?P<transactions>\d+).*\n',
r'\s+deadlocks:\s+(?P<deadlocks>\d+).*\n',
r'\s+total time:\s+(?P<duration>[\d\.]+).*\n',
]
TRANSFORM_FIELDS = {
'queries_read': int,
'queries_write': int,
'queries_other': int,
'queries_total': int,
'duration': float,
'transactions': int,
'deadlocks': int,
}
def parse_sysbench_oltp(raw):
result = {}
for pattern in PATTERNS:
for parsed in re.finditer(pattern, raw):
result.update(parsed.groupdict())
for k in result.keys():
if k in TRANSFORM_FIELDS:
result[k] = TRANSFORM_FIELDS[k](result[k])
return result
class SysbenchOltpExecutor(base.BaseExecutor):
def get_command(self):
cmd = base.CommandLine('sysbench')
cmd.add('--test', 'oltp')
cmd.add('--db-driver', 'mysql')
cmd.add('--mysql-table-engine', 'innodb')
cmd.add('--mysql-engine-trx', 'yes')
cmd.add('--num-threads', self.test_definition.get('threads') or 10)
cmd.add('--max-time', self.get_expected_duration())
cmd.add('--max-requests', 0)
cmd.add('--mysql-host', 'localhost')
cmd.add('--mysql-db', 'sbtest')
cmd.add('--oltp-table-name', 'sbtest')
cmd.add('--oltp-table-size',
self.test_definition.get('table_size') or 100000)
# cmd.add('--oltp-num-tables',
# self.test_definition.get('num_tables') or 10)
# cmd.add('--oltp-auto-inc', 'off')
# cmd.add('--oltp-read-only', 'off')
cmd.add('run')
return cmd.make()
def process_reply(self, record):
stdout = record.get('stdout')
return parse_sysbench_oltp(stdout)

View File

@ -0,0 +1,28 @@
Sysbench Report
---------------
This is the report of execution test plan
:ref:`sql_db_test_plan` with `Sysbench`_ tool.
Results
^^^^^^^
Chart and table:
{{'''
title: Queries per second
axes:
x: threads
y: queries per sec
chart: line
pipeline:
- { $match: { class: sysbench-oltp, status: OK }}
- { $group: { _id: { threads: "$threads" }, queries_total_per_sec: { $avg: { $divide: ["$queries_total", "$duration"] }}}}
- { $project: { x: "$_id.threads", y: "$queries_total_per_sec" }}
- { $sort: { x: 1 }}
''' | chart
}}
.. references:
.. _Sysbench: https://github.com/akopytov/sysbench

View File

@ -0,0 +1,25 @@
title: DB
description:
This scenario uses sysbench to execute DB test plan.
preparation:
ansible-playbook:
-
tasks:
- apt: name=sysbench
become: yes
become_user: root
become_method: sudo
execution:
matrix:
threads: [ 10, 20, 30, 40, 50, 60 ]
tests:
-
title: sysbench-oltp
class: sysbench-oltp
time: 10
report:
template: sysbench.rst

View File

@ -0,0 +1,77 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import testtools
from performa.executors import sysbench
OLTP_OUTPUT = '''
sysbench 0.4.12: multi-threaded system evaluation benchmark
Running the test with following options:
Number of threads: 20
Doing OLTP test.
Running mixed OLTP test
Using Special distribution
Using "BEGIN" for starting transactions
Not using auto_inc on the id column
Threads started!
Time limit exceeded, exiting...
(last message repeated 19 times)
Done.
OLTP test statistics:
queries performed:
read: 9310
write: 3325
other: 1330
total: 13965
transactions: 665 (10.94 per sec.)
deadlocks: 0 (0.00 per sec.)
read/write requests: 12635 (207.79 per sec.)
other operations: 1330 (21.87 per sec.)
Test execution summary:
total time: 60.8074s
total number of events: 665
total time taken by event execution: 1208.0577
per-request statistics:
min: 876.31ms
avg: 1816.63ms
max: 3792.73ms
approx. 95 percentile: 2886.19ms
Threads fairness:
events (avg/stddev): 33.2500/0.70
execution time (avg/stddev): 60.4029/0.21
'''
class TestUtils(testtools.TestCase):
def test_parse_oltp(self):
expected = {
'version': '0.4.12',
'queries_read': 9310,
'queries_write': 3325,
'queries_other': 1330,
'queries_total': 13965,
'transactions': 665,
'deadlocks': 0,
'duration': 60.8074,
}
self.assertEqual(expected, sysbench.parse_sysbench_oltp(OLTP_OUTPUT))

View File

@ -0,0 +1,101 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from performa.engine import utils
class TestUtils(testtools.TestCase):
@mock.patch('os.walk')
@mock.patch('performa.engine.utils.resolve_relative_path')
def test_make_help_options(self, resolve_mock, walk_mock):
base_dir = 'abc/def'
abs_dir = '/files/' + base_dir
walk_mock.side_effect = [
[(abs_dir, [], ['klm.yaml']), (abs_dir, [], ['ijk.yaml'])],
]
resolve_mock.side_effect = [abs_dir]
expected = 'List: "ijk", "klm"'
observed = utils.make_help_options('List: %s', base_dir)
self.assertEqual(expected, observed)
@mock.patch('os.walk')
@mock.patch('performa.engine.utils.resolve_relative_path')
def test_make_help_options_subdir(self, resolve_mock, walk_mock):
base_dir = 'abc/def'
abs_dir = '/files/' + base_dir
walk_mock.side_effect = [
[(abs_dir + '/sub', [], ['klm.yaml']),
(abs_dir + '/sub', [], ['ijk.yaml'])],
]
resolve_mock.side_effect = [abs_dir]
expected = 'List: "sub/ijk", "sub/klm"'
observed = utils.make_help_options('List: %s', base_dir)
self.assertEqual(expected, observed)
@mock.patch('os.walk')
@mock.patch('performa.engine.utils.resolve_relative_path')
def test_make_help_options_with_filter(self, resolve_mock, walk_mock):
base_dir = 'abc/def'
abs_dir = '/files/' + base_dir
walk_mock.side_effect = [
[(abs_dir + '/sub', [], ['klm.yaml']),
(abs_dir + '/sub', [], ['ijk.html']),
(abs_dir + '/sub', [], ['mno.yaml'])],
]
resolve_mock.side_effect = [abs_dir]
expected = 'List: "sub/klm", "sub/mno"'
observed = utils.make_help_options(
'List: %s', base_dir, type_filter=lambda x: x.endswith('.yaml'))
self.assertEqual(expected, observed)
def test_algebraic_product_empty(self):
expected = [{}]
observed = list(utils.algebraic_product())
self.assertEqual(expected, observed)
def test_algebraic_product_string(self):
expected = [{'a': 1, 'b': 'zebra'}, {'a': 2, 'b': 'zebra'}]
observed = list(utils.algebraic_product(a=[1, 2], b='zebra'))
self.assertEqual(expected, observed)
def test_algebraic_product_number(self):
expected = [{'a': 'x', 'b': 4}, {'a': 2, 'b': 4}]
observed = list(utils.algebraic_product(a=['x', 2], b=4))
self.assertEqual(expected, observed)
def test_strict(self):
self.assertEqual('some_01_string_a',
utils.strict('Some 01-string (brr!) + %% A'))
def test_parse_url(self):
self.assertEqual(dict(host='aa', port=123),
utils.parse_url('aa:123'))
def test_parse_url_host_only(self):
self.assertEqual(dict(host='aa'),
utils.parse_url('aa'))

View File

@ -3,3 +3,15 @@
# process, which may cause wedges in the gate later.
pbr>=1.6
ansible
iso8601>=0.1.9
oslo.config>=2.6.0 # Apache-2.0
oslo.i18n>=1.5.0 # Apache-2.0
oslo.log>=1.12.0 # Apache-2.0
oslo.serialization>=1.10.0 # Apache-2.0
oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0
pygal
pymongo
PyYAML>=3.1.0
six>=1.9.0

View File

@ -5,9 +5,10 @@ description-file =
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
home-page = https://github.com/openstack/performance-docs
classifier =
Environment :: OpenStack
Intended Audience :: Developers
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
@ -15,14 +16,20 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
[files]
packages =
performa
[entry_points]
console_scripts =
performa = performa.engine.main:main
performa-report = performa.engine.report:main
oslo.config.opts =
oslo_log = oslo_log._options:list_opts
performa.engine.config = performa.engine.config:list_opts
[build_sphinx]
source-dir = doc/source
build-dir = doc/build

View File

@ -6,6 +6,7 @@ hacking<0.11,>=0.10.0
coverage>=3.6
python-subunit>=0.0.18
rst2pdf
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
oslosphinx>=2.5.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0

38
tox.ini
View File

@ -1,6 +1,6 @@
[tox]
minversion = 2.0
envlist = py34-constraints,py27-constraints,pypy-constraints,pep8-constraints
envlist = docs,py27,pep8
minversion = 1.6
skipsdist = True
[testenv]
@ -13,47 +13,17 @@ setenv =
deps = -r{toxinidir}/test-requirements.txt
commands = python setup.py test --slowest --testr-args='{posargs}'
[testenv:common-constraints]
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
[testenv:pep8]
commands = flake8 {posargs}
[testenv:pep8-constraints]
install_command = {[testenv:common-constraints]install_command}
commands = flake8 {posargs}
[testenv:venv]
commands = {posargs}
[testenv:venv-constraints]
install_command = {[testenv:common-constraints]install_command}
commands = {posargs}
[testenv:cover]
commands = python setup.py test --coverage --testr-args='{posargs}'
[testenv:cover-constraints]
install_command = {[testenv:common-constraints]install_command}
commands = python setup.py test --coverage --testr-args='{posargs}'
[testenv:pep8]
commands = flake8
[testenv:docs]
commands = python setup.py build_sphinx
[testenv:docs-constraints]
install_command = {[testenv:common-constraints]install_command}
commands = python setup.py build_sphinx
[testenv:debug]
commands = oslo_debug_helper {posargs}
[testenv:debug-constraints]
install_command = {[testenv:common-constraints]install_command}
commands = oslo_debug_helper {posargs}
[flake8]
# E123, E125 skipped as they are invalid PEP-8.
show-source = True
ignore = E123,E125
builtins = _