Add omsimulator multihost scenario

This commit is contained in:
Ilya Shakhat 2016-03-17 16:37:26 +03:00
parent 95761686cd
commit b11a1eb147
6 changed files with 244 additions and 307 deletions

View File

@ -117,6 +117,8 @@ def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder,
report_definition = scenario['report']
report_template = report_definition['template']
LOG.info('Using report template: %s', report_template)
_make_dir(doc_folder)
jinja_env = jinja2.Environment()
@ -137,6 +139,15 @@ def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder,
LOG.info('The report is written to %s', doc_folder)
def resolve_vars(scenario_template, vars):
jinja_env = jinja2.Environment()
compiled_template = jinja_env.from_string(scenario_template)
rendered_template = compiled_template.render(vars)
return rendered_template
def main():
utils.init_config_and_logging(config.MAIN_OPTS)
@ -144,7 +155,9 @@ def main():
cfg.CONF.scenario,
alias_mapper=lambda f: config.SCENARIOS + '%s.yaml' % f)
scenario = utils.read_yaml_file(scenario_file_path)
scenario_raw = utils.read_file(scenario_file_path)
scenario_raw = resolve_vars(scenario_raw, cfg.CONF.vars)
scenario = yaml.safe_load(scenario_raw)
base_dir = os.path.dirname(scenario_file_path)
generate_report(scenario, base_dir, cfg.CONF.mongo_url, cfg.CONF.mongo_db,

View File

@ -1,9 +1,9 @@
#!/usr/bin/python
import copy
import os
import tempfile
import random
import signal
import tempfile
SERVER_PID = os.path.join(tempfile.gettempdir(), 'performa.oms.pid')
SERVER_FILE_NAME = os.path.join(tempfile.gettempdir(), 'performa.oms.srv')
@ -104,16 +104,20 @@ def run(module):
client_tool = 'rpc-client'
params['python'] = PYTHON
# todo: fix topic support in omsimulator
# params['topic'] = 'performa-%d' % (random.random() * 1000000)
params['server_tool'] = server_tool
params['client_tool'] = client_tool
params['server_file'] = SERVER_FILE_NAME
params['client_file'] = CLIENT_FILE_NAME
server = ('%(python)s simulator.py '
# '--topic %(topic)s '
'--url %(url)s '
'--json %(server_file)s '
'%(server_tool)s ') % params
client = ('%(python)s simulator.py '
# '--topic %(topic)s '
'--url=%(url)s '
'--json %(client_file)s '
'-l %(duration)s '

View File

@ -1,211 +0,0 @@
Oslo.messaging simulator report
-------------------------------
This report shows how many concurrent threads can handle a single
oslo.messaging process.
Test Case 1: RPC CALL Throughput Test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Message processing**
Messages are collected at 3 points: ``sent`` - messages sent by the client,
``received`` - messages received by the server, ``round-trip`` - replies
received by the client. Also the number of lost messages is calculated.
{{'''
title: RPC CALL Message count
axes:
x: threads
y: sent, msg
y2: received, msg
y3: round-trip, msg
y4: lost, msg
chart: line
pipeline:
- { $match: { task: omsimulator, mode: call }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
sent: { $avg: "$client.count" },
received: { $avg: "$server.count" },
round_trip: { $avg: "$round_trip.count" },
lost: { $avg: { $subtract: ["$client.count", "$round_trip.count"] }}
}}
- { $project: { x: "$_id.threads",
y: "$sent",
y2: "$received",
y3: "$round_trip",
y4: "$lost"
}}
- { $sort: { x: 1 }}
''' | chart
}}
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
depending on number of concurrent threads.
{{'''
title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count
axes:
x: threads
y: sent, msg/sec
y2: received, msg/sec
y3: round-trip, msg/sec
y4: latency, ms
y5: RabbitMQ CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator, mode: call }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }},
msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }},
msg_round_trip_per_sec: { $avg: { $divide: ["$round_trip.count", "$round_trip.duration"] }},
latency: { $avg: "$round_trip.latency" },
rabbit_total: { $avg: "$rabbit_total" }
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: "$msg_received_per_sec",
y3: "$msg_round_trip_per_sec",
y4: { $multiply: [ "$latency", 1000 ] },
y5: { $multiply: [ "$rabbit_total", 100 ] }
}}
- { $sort: { x: 1 }}
''' | chart
}}
Test Case 2: RPC CAST Throughput Test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Message processing**
Messages are collected at 2 points: ``sent`` - messages sent by the client
and ``received`` - messages received by the server. Also the number of lost
messages is calculated.
{{'''
title: RPC CAST Message count
axes:
x: threads
y: sent, msg
y2: received, msg
y3: lost, msg
chart: line
pipeline:
- { $match: { task: omsimulator, mode: cast }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
sent: { $avg: "$client.count" },
received: { $avg: "$server.count" },
lost: { $avg: { $subtract: ["$client.count", "$server.count"] }}
}}
- { $project: { x: "$_id.threads",
y: "$sent",
y2: "$received",
y3: "$lost"
}}
- { $sort: { x: 1 }}
''' | chart
}}
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
depending on number of concurrent threads.
{{'''
title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count
axes:
x: threads
y: sent, msg/sec
y2: received, msg/sec
y3: latency, ms
y4: RabbitMQ CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator, mode: cast }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }},
msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }},
latency: { $avg: "$server.latency" },
rabbit_total: { $avg: "$rabbit_total" }
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: "$msg_received_per_sec",
y3: { $multiply: [ "$latency", 1000 ] },
y4: { $multiply: [ "$rabbit_total", 100 ] }
}}
- { $sort: { x: 1 }}
''' | chart
}}
Test Case 3: Notification Throughput Test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Message processing**
Messages are collected at 2 points: ``sent`` - messages sent by the client
and ``received`` - messages received by the server. Also the number of lost
messages is calculated.
{{'''
title: NOTIFY Message count
axes:
x: threads
y: sent, msg
y2: received, msg
l3: lost, msg
chart: line
pipeline:
- { $match: { task: omsimulator, mode: notify }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
sent: { $avg: "$client.count" },
received: { $avg: "$server.count" },
lost: { $avg: { $subtract: ["$client.count", "$server.count"] }}
}}
- { $project: { x: "$_id.threads",
y: "$sent",
y2: "$received",
y3: "$lost"
}}
- { $sort: { x: 1 }}
''' | chart
}}
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
depending on number of concurrent threads.
{{'''
title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count
axes:
x: threads
y: sent, msg/sec
y2: received, msg/sec
y3: latency, ms
y4: RabbitMQ CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator, mode: notify }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }},
msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }},
latency: { $avg: "$server.latency" },
rabbit_total: { $avg: "$rabbit_total" }
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: "$msg_received_per_sec",
y3: { $multiply: [ "$latency", 1000 ] },
y4: { $multiply: [ "$rabbit_total", 100 ] }
}}
- { $sort: { x: 1 }}
''' | chart
}}

View File

@ -47,7 +47,7 @@ execution:
-
hosts: {{ tester_hosts }}
matrix:
threads: [ 1, 2, 5, 10, 20, 50, 100 ]
threads: [ 1, 2, 5, 10, 20, 50 ]
tasks:
- omsimulator:
mode: call
@ -56,7 +56,7 @@ execution:
-
hosts: {{ tester_hosts }}
matrix:
threads: [ 1, 2, 5, 10, 20, 50, 100 ]
threads: [ 1, 2, 5, 10, 20, 50 ]
tasks:
- omsimulator:
mode: cast
@ -65,7 +65,7 @@ execution:
-
hosts: {{ tester_hosts }}
matrix:
threads: [ 1, 2, 5, 10, 20, 50, 100 ]
threads: [ 1, 2, 5, 10, 20, 50 ]
tasks:
- omsimulator:
mode: notify
@ -89,4 +89,4 @@ aggregation:
- { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
report:
template: omsimulator-threading.rst
template: omsimulator.rst

View File

@ -1,106 +1,217 @@
OMSimulator Report
------------------
Oslo.messaging simulator report
-------------------------------
This is the report of execution test plan
:ref:`mq_test_plan` with `OMSimulator`_ tool.
This report is result of `message_queue_performance`_ execution
with `Oslo.messaging Simulator`_
Results
^^^^^^^
Test Case 1: RPC CALL Throughput Test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Message processing**
The chart and table show the number of messages processed by a single process
depending on number of eventlet threads inside of it. Messages are collected
at 3 points: ``sent`` - messages sent by the client, ``received`` - messages
received by the server, ``round-trip`` - replies received by the client.
Messages are collected at 3 points: ``sent`` - messages sent by the client,
``received`` - messages received by the server, ``round-trip`` - replies
received by the client. Also the number of lost messages is calculated.
{{'''
title: Throughput
title: RPC CALL Message count
axes:
x: threads
y: sent, msg
y2: received, msg
y3: round-trip, msg
y4: lost, msg
chart: line
pipeline:
- { $match: { task: omsimulator, component: client }}
- { $match: { task: omsimulator, mode: call }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
sent: { $avg: "$client.count" },
received: { $avg: "$server.count" },
round_trip: { $avg: "$round_trip.count" }
sent: { $sum: "$client.count" },
received: { $sum: "$server.count" },
round_trip: { $sum: "$round_trip.count" },
lost: { $sum: { $subtract: ["$client.count", "$round_trip.count"] }}
}}
- { $project: { x: "$_id.threads",
y: "$sent",
y2: "$received",
y3: "$round_trip"
y3: "$round_trip",
y4: "$lost"
}}
- { $sort: { x: 1 }}
''' | chart
}}
**Message throughput, latency depending on thread count**
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
depending on number of concurrent threads.
{{'''
title: Throughput, latency depending on thread count
title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count
axes:
x: threads
y: throughput, msg/sec
y2: latency, ms
y: sent, msg/sec
y2: received, msg/sec
y3: round-trip, msg/sec
y4: latency, ms
y5: RabbitMQ CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator, component: client }}
- { $match: { task: omsimulator, mode: call }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$count", "$duration"] }},
latency: { $avg: "$latency" },
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: { $multiply: ["$latency", 1000] }
}}
- { $sort: { x: 1 }}
''' | chart
}}
Messages per second and rabbit CPU consumption depending on threads count:
{{'''
title: Queries and and CPU util per second
axes:
x: threads
y: queries per sec
y2: rabbit CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$count", "$duration"] }},
msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }},
msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }},
msg_round_trip_per_sec: { $avg: { $divide: ["$round_trip.count", "$round_trip.duration"] }},
latency: { $avg: "$round_trip.latency" },
rabbit_total: { $avg: "$rabbit_total" }
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: { $multiply: [ "$rabbit_total", 100 ] }
y2: "$msg_received_per_sec",
y3: "$msg_round_trip_per_sec",
y4: { $multiply: [ "$latency", 1000 ] },
y5: { $multiply: [ "$rabbit_total", 100 ] }
}}
- { $sort: { x: 1 }}
''' | chart
}}
Test Case 2: RPC CAST Throughput Test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Message processing**
Messages are collected at 2 points: ``sent`` - messages sent by the client
and ``received`` - messages received by the server. Also the number of lost
messages is calculated.
{{'''
title: Latency depending on msg/sec
title: RPC CAST Message count
axes:
x: messages per sec
y: latency
x: threads
y: sent, msg
y2: received, msg
y3: lost, msg
chart: line
pipeline:
- { $match: { task: omsimulator }}
- { $match: { task: omsimulator, mode: cast }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$count", "$duration"] }},
latency: { $avg: "$latency" }
sent: { $sum: "$client.count" },
received: { $sum: "$server.count" },
lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
}}
- { $project: { x: "$msg_sent_per_sec",
y: { $multiply: ["$latency", 1000] }
- { $project: { x: "$_id.threads",
y: "$sent",
y2: "$received",
y3: "$lost"
}}
- { $sort: { x: 1 }}
''' | chart
}}
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
depending on number of concurrent threads.
{{'''
title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count
axes:
x: threads
y: sent, msg/sec
y2: received, msg/sec
y3: latency, ms
y4: RabbitMQ CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator, mode: cast }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }},
msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }},
latency: { $avg: "$server.latency" },
rabbit_total: { $avg: "$rabbit_total" }
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: "$msg_received_per_sec",
y3: { $multiply: [ "$latency", 1000 ] },
y4: { $multiply: [ "$rabbit_total", 100 ] }
}}
- { $sort: { x: 1 }}
''' | chart
}}
Test Case 3: Notification Throughput Test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Message processing**
Messages are collected at 2 points: ``sent`` - messages sent by the client
and ``received`` - messages received by the server. Also the number of lost
messages is calculated.
{{'''
title: NOTIFY Message count
axes:
x: threads
y: sent, msg
y2: received, msg
y3: lost, msg
chart: line
pipeline:
- { $match: { task: omsimulator, mode: notify }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
sent: { $sum: "$client.count" },
received: { $sum: "$server.count" },
lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
}}
- { $project: { x: "$_id.threads",
y: "$sent",
y2: "$received",
y3: "$lost"
}}
- { $sort: { x: 1 }}
''' | chart
}}
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
depending on number of concurrent threads.
{{'''
title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count
axes:
x: threads
y: sent, msg/sec
y2: received, msg/sec
y3: latency, ms
y4: RabbitMQ CPU consumption, %
chart: line
pipeline:
- { $match: { task: omsimulator, mode: notify }}
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }},
msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }},
latency: { $avg: "$server.latency" },
rabbit_total: { $avg: "$rabbit_total" }
}}
- { $project: { x: "$_id.threads",
y: "$msg_sent_per_sec",
y2: "$msg_received_per_sec",
y3: { $multiply: [ "$latency", 1000 ] },
y4: { $multiply: [ "$rabbit_total", 100 ] }
}}
- { $sort: { x: 1 }}
''' | chart
}}
.. references:
.. _message_queue_performance: http://docs.openstack.org/developer/performance-docs/test_plans/mq/plan.html
.. _Oslo.messaging Simulator: https://github.com/openstack/oslo.messaging/blob/master/tools/simulator.py

View File

@ -4,62 +4,82 @@ description:
This scenario uses oslo.messaging simulator tool to execute MQ test plan.
parameters:
target: List of hosts were omsimulator will be executed
tester_hosts: List of hosts were omsimulator will be executed
rabbit_hosts: List of hosts were RabbitMQ runs
rabbit_url: RabbitMQ address
setup:
-
hosts: {{ target }}
hosts: {{ tester_hosts }}
tasks:
- apt: name=git
become: yes
# - name: installing omsimulator
# git: repo=git://git.openstack.org/openstack/oslo.messaging
# dest=/tmp/performa/oslo.messaging
# - name: installing omsimulator
# git: repo=git://git.openstack.org/openstack/oslo.messaging
# dest=/tmp/performa/oslo.messaging
# - command: git fetch https://review.openstack.org/openstack/oslo.messaging refs/changes/91/291191/2
# args:
# chdir: /tmp/performa/oslo.messaging
# - command: git checkout FETCH_HEAD
# args:
# chdir: /tmp/performa/oslo.messaging
# - apt: name=atop
# become: yes
# - apt: name=daemon
# become: yes
- apt: name=daemon
become: yes
- name: installing omsimulator
git: repo=git://git.openstack.org/openstack/oslo.messaging
dest=/tmp/performa/oslo.messaging
- apt: name=python-dev
become: yes
- apt: name=python-pip
become: yes
- pip: name=virtualenv
become: yes
- pip: requirements=/tmp/performa/oslo.messaging/requirements.txt virtualenv=/tmp/performa/oslo.messaging/.venv
- pip: name=eventlet virtualenv=/tmp/performa/oslo.messaging/.venv
- command: /tmp/performa/oslo.messaging/.venv/bin/python setup.py install
args:
chdir: /tmp/performa/oslo.messaging
-
hosts: {{ rabbit_hosts }}
tasks:
- apt: name=atop
become: yes
- apt: name=daemon
become: yes
execution:
-
hosts: {{ target }}
hosts: {{ rabbit_hosts }}
tasks:
- atop: command=start
# -
# hosts: {{ target }}
# matrix:
# threads: [ 1, 2, 5, 10, 25, 50, 100 ]
# tasks:
# - omsimulator:
# mode: call
# duration: 10
# url: "rabbit://stackrabbit:swordfish@localhost:5672/"
-
hosts: {{ target }}
hosts: {{ tester_hosts }}
matrix:
host_count: [ 1 ]
threads: [ 1, 10, 100 ]
host_count: [ 1, 2, 5, 10 ]
tasks:
- omsimulator:
mode: call
duration: 10
duration: 100
threads: 10
url: {{ rabbit_url }}
-
hosts: {{ target }}
hosts: {{ tester_hosts }}
matrix:
host_count: [ 1, 2, 5, 10 ]
tasks:
- omsimulator:
mode: cast
duration: 100
threads: 10
url: {{ rabbit_url }}
-
hosts: {{ tester_hosts }}
matrix:
host_count: [ 1, 2, 5, 10 ]
tasks:
- omsimulator:
mode: notify
duration: 100
threads: 10
url: {{ rabbit_url }}
-
hosts: {{ rabbit_hosts }}
tasks:
- atop:
command: stop
labels: [ CPU, PRC, PRM ]
labels: [ PRC ]
aggregation:
-
@ -68,7 +88,7 @@ aggregation:
{ task: omsimulator }
values:
pipeline:
- { $match: { task: atop, status: OK, label: PRC, name: beam.smp }}
- { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
- { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
report: