Run multiple omsimulator pairs on a single node
This commit is contained in:
Ilya Shakhat
parent
81ba681918
commit
67c1957349
|
|
@ -1,5 +1,6 @@
|
|||
#!/usr/bin/python
|
||||
import copy
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import tempfile
|
||||
|
|
@ -11,35 +12,6 @@ UNIQUE_NAME = 'performa_omsimulator'
|
|||
DIR = '/tmp/performa/oslo.messaging/tools/'
|
||||
PYTHON = '/tmp/performa/oslo.messaging/.venv/bin/python'
|
||||
|
||||
PATTERNS = [
|
||||
r'(?P<msg_sent>\d+) messages were sent for (?P<duration>\d+) sec',
|
||||
r'(?P<bytes_sent>\d+) bytes were sent for',
|
||||
]
|
||||
TRANSFORM_FIELDS = {
|
||||
'msg_sent': int,
|
||||
'bytes_sent': int,
|
||||
'duration': int,
|
||||
}
|
||||
|
||||
|
||||
def parse_output(raw):
|
||||
result = {}
|
||||
|
||||
for pattern in PATTERNS:
|
||||
for parsed in re.finditer(pattern, raw):
|
||||
result.update(parsed.groupdict())
|
||||
|
||||
for k in result.keys():
|
||||
if k in TRANSFORM_FIELDS:
|
||||
result[k] = TRANSFORM_FIELDS[k](result[k])
|
||||
|
||||
result['msg_sent_bandwidth'] = (result.get('msg_sent', 0) /
|
||||
result.get('duration', 1))
|
||||
result['bytes_sent_bandwidth'] = (result.get('bytes_sent', 0) /
|
||||
result.get('duration', 1))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def chdir(module):
|
||||
try:
|
||||
|
|
@ -48,29 +20,6 @@ def chdir(module):
|
|||
module.fail_json(msg='Failed to change dir to %s: %s' % (DIR, e))
|
||||
|
||||
|
||||
def start_daemon(module, cmd):
|
||||
cmd = ('daemon -n %(name)s -D %(dir)s -F %(pid)s -U -- %(cmd)s' %
|
||||
dict(name=UNIQUE_NAME, dir=DIR, pid=SERVER_PID, cmd=cmd))
|
||||
|
||||
rc, stdout, stderr = module.run_command(cmd)
|
||||
result = dict(changed=True, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
|
||||
|
||||
if rc:
|
||||
module.fail_json(msg='Failed to start omsimulator', **result)
|
||||
|
||||
|
||||
def stop_daemon(module):
|
||||
rc, stdout, stderr = module.run_command('/bin/cat %s' % SERVER_PID)
|
||||
|
||||
if rc:
|
||||
return
|
||||
|
||||
rc, stdout, stderr = module.run_command('pgrep -P %s' % stdout)
|
||||
os.kill(int(stdout), signal.SIGINT)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def read_file(filename):
|
||||
fd = None
|
||||
try:
|
||||
|
|
@ -92,35 +41,16 @@ def transform_series(series):
|
|||
return result
|
||||
|
||||
|
||||
def run(module):
|
||||
params = copy.deepcopy(module.params)
|
||||
def make_file_name(base, index):
|
||||
return '%s-%s' % (base, index)
|
||||
|
||||
if params['mode'] == 'notify':
|
||||
server_tool = 'notify-server'
|
||||
client_tool = 'notify-client'
|
||||
else:
|
||||
server_tool = 'rpc-server'
|
||||
client_tool = 'rpc-client'
|
||||
|
||||
params['python'] = PYTHON
|
||||
# todo: fix topic support in omsimulator
|
||||
# params['topic'] = 'performa-%d' % (random.random() * 1000000)
|
||||
params['server_tool'] = server_tool
|
||||
params['client_tool'] = client_tool
|
||||
params['server_file'] = SERVER_FILE_NAME
|
||||
params['client_file'] = CLIENT_FILE_NAME
|
||||
|
||||
params['url'] = params['server_url'] or params['url']
|
||||
server = ('%(python)s simulator.py '
|
||||
# '--topic %(topic)s '
|
||||
'--url %(url)s '
|
||||
'--json %(server_file)s '
|
||||
'%(server_tool)s ') % params
|
||||
|
||||
def make_client_cmd(params, i):
|
||||
params['client_file'] = make_file_name(CLIENT_FILE_NAME, i)
|
||||
params['url'] = params['client_url'] or params['url']
|
||||
|
||||
client = ('%(python)s simulator.py '
|
||||
# '--topic %(topic)s '
|
||||
'--url=%(url)s '
|
||||
'--url %(url)s '
|
||||
'--json %(client_file)s '
|
||||
'-l %(duration)s '
|
||||
'%(client_tool)s '
|
||||
|
|
@ -134,45 +64,121 @@ def run(module):
|
|||
if params['mode'] == 'fanout':
|
||||
client += '--is-fanout True '
|
||||
|
||||
start_daemon(module, server)
|
||||
return client
|
||||
|
||||
rc, stdout, stderr = module.run_command(client)
|
||||
|
||||
if rc:
|
||||
module.fail_json(msg='Failed to start omsimulator',
|
||||
stderr=stderr, rc=rc, cmd=client)
|
||||
def make_server_cmd(params, i):
|
||||
params['server_file'] = make_file_name(SERVER_FILE_NAME, i)
|
||||
params['url'] = params['server_url'] or params['url']
|
||||
|
||||
stop_daemon(module)
|
||||
server = ('%(python)s simulator.py '
|
||||
'--url %(url)s '
|
||||
'--json %(server_file)s '
|
||||
'%(server_tool)s ') % params
|
||||
|
||||
try:
|
||||
client_data = read_file(CLIENT_FILE_NAME)
|
||||
server_data = read_file(SERVER_FILE_NAME)
|
||||
return server
|
||||
|
||||
|
||||
def run_client(module, command):
|
||||
module.run_command(command)
|
||||
|
||||
|
||||
def run_client_pool(module, params):
|
||||
processes = []
|
||||
|
||||
for i in range(params['processes']):
|
||||
cmd = make_client_cmd(params, i)
|
||||
p = multiprocessing.Process(target=run_client, args=(module, cmd))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
|
||||
def run_server(module, command):
|
||||
module.run_command(command)
|
||||
|
||||
|
||||
def start_server_pool(module, params):
|
||||
processes = []
|
||||
|
||||
for i in range(params['processes']):
|
||||
cmd = make_server_cmd(params, i)
|
||||
p = multiprocessing.Process(target=run_client, args=(module, cmd))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
|
||||
return processes
|
||||
|
||||
|
||||
def stop_server_pool(module, processes):
|
||||
for p in processes:
|
||||
rc, stdout, stderr = module.run_command('pgrep -P %s' % p.pid)
|
||||
|
||||
for child in (int(p) for p in stdout.split('\n') if p):
|
||||
os.kill(child, signal.SIGINT)
|
||||
|
||||
time.sleep(3) # let simulators handle the signal
|
||||
|
||||
for p in processes:
|
||||
os.kill(p.pid, signal.SIGINT)
|
||||
p.join()
|
||||
|
||||
|
||||
def collect_results(params):
|
||||
result = dict(records=[], series=[])
|
||||
|
||||
for i in range(params['processes']):
|
||||
client_data = read_file(make_file_name(CLIENT_FILE_NAME, i))
|
||||
server_data = read_file(make_file_name(SERVER_FILE_NAME, i))
|
||||
|
||||
client_summary = client_data['summary']['client']
|
||||
|
||||
record = dict(start=client_summary['start'],
|
||||
end=client_summary['end'],
|
||||
record = dict(start=client_summary['start'], end=client_summary['end'],
|
||||
client=client_summary)
|
||||
|
||||
if 'round_trip' in client_data['summary']:
|
||||
round_trip_summary = client_data['summary']['round_trip']
|
||||
record['round_trip'] = round_trip_summary
|
||||
|
||||
if 'error' in client_data['summary']:
|
||||
error_summary = client_data['summary']['error']
|
||||
record['error'] = error_summary
|
||||
|
||||
server_summary = server_data['summary']
|
||||
record['server'] = server_summary
|
||||
|
||||
series = transform_series(client_data['series'])
|
||||
series.extend(transform_series(server_data['series']))
|
||||
|
||||
result = dict(records=[record], series=series)
|
||||
result['records'].append(record)
|
||||
result['series'] += series
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def run(module):
|
||||
params = copy.deepcopy(module.params)
|
||||
|
||||
if params['mode'] == 'notify':
|
||||
server_tool = 'notify-server'
|
||||
client_tool = 'notify-client'
|
||||
else:
|
||||
server_tool = 'rpc-server'
|
||||
client_tool = 'rpc-client'
|
||||
|
||||
params['python'] = PYTHON
|
||||
params['server_tool'] = server_tool
|
||||
params['client_tool'] = client_tool
|
||||
|
||||
server_processes = start_server_pool(module, params)
|
||||
|
||||
run_client_pool(module, params)
|
||||
|
||||
stop_server_pool(module, server_processes)
|
||||
|
||||
try:
|
||||
result = collect_results(params)
|
||||
module.exit_json(**result)
|
||||
except Exception as e:
|
||||
msg = 'Failed to read omsimulator output: %s' % e
|
||||
module.fail_json(msg=msg, rc=rc, stderr=stderr, stdout=stdout)
|
||||
module.fail_json(msg=msg)
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -184,6 +190,7 @@ def main():
|
|||
client_url=dict(),
|
||||
server_url=dict(),
|
||||
threads=dict(type='int', default=10),
|
||||
processes=dict(type='int', default=1),
|
||||
duration=dict(type='int', default=10),
|
||||
timeout=dict(type='int', default=5),
|
||||
sending_delay=dict(type='float', default=-1.0),
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ execution:
|
|||
tasks:
|
||||
- atop:
|
||||
command: stop
|
||||
labels: [ PRC ]
|
||||
labels: [ PRC, PRM ]
|
||||
|
||||
aggregation:
|
||||
-
|
||||
|
|
@ -93,8 +93,54 @@ aggregation:
|
|||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
|
||||
- { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
|
||||
{% if rabbit_hosts[1] %}
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
{% if rabbit_hosts[2] %}
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
report:
|
||||
template: omsimulator.rst
|
||||
|
|
|
|||
|
|
@ -8,11 +8,14 @@ with `Oslo.messaging Simulator`_
|
|||
Test Case 1: RPC CALL Throughput Test
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
**Message processing**
|
||||
Message processing
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Messages are collected at 3 points: ``sent`` - messages sent by the client,
|
||||
``received`` - messages received by the server, ``round-trip`` - replies
|
||||
received by the client. Also the number of lost messages is calculated.
|
||||
Sizes of messages is based on the distribution of messages collected on
|
||||
the 100-node cloud.
|
||||
|
||||
{{'''
|
||||
title: RPC CALL Message count
|
||||
|
|
@ -25,7 +28,7 @@ received by the client. Also the number of lost messages is calculated.
|
|||
chart: line
|
||||
pipeline:
|
||||
- { $match: { task: omsimulator, mode: call }}
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
|
||||
sent: { $sum: "$client.count" },
|
||||
received: { $sum: "$server.count" },
|
||||
round_trip: { $sum: "$round_trip.count" },
|
||||
|
|
@ -42,7 +45,8 @@ received by the client. Also the number of lost messages is calculated.
|
|||
}}
|
||||
|
||||
|
||||
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
|
||||
The throughput, latency and RabbitMQ CPU utilization
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
|
||||
depending on number of concurrent threads.
|
||||
|
|
@ -58,22 +62,89 @@ depending on number of concurrent threads.
|
|||
y5: RabbitMQ CPU consumption, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- { $match: { task: omsimulator, mode: call }}
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
|
||||
msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }},
|
||||
latency: { $avg: "$round_trip.latency" },
|
||||
rabbit_total: { $avg: "$rabbit_total" }
|
||||
}}
|
||||
- { $project: { x: "$_id.threads",
|
||||
y: "$msg_sent_per_sec",
|
||||
y2: "$msg_received_per_sec",
|
||||
y3: "$msg_round_trip_per_sec",
|
||||
y4: { $multiply: [ "$latency", 1000 ] },
|
||||
y5: { $multiply: [ "$rabbit_total", 100 ] }
|
||||
}}
|
||||
- { $sort: { x: 1 }}
|
||||
- $match: { task: omsimulator, mode: call }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
||||
msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
|
||||
latency: { $avg: "$round_trip.latency" }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y: "$msg_sent_per_sec"
|
||||
y2: "$msg_received_per_sec"
|
||||
y3: "$msg_round_trip_per_sec"
|
||||
y4: { $multiply: [ "$latency", 1000 ] }
|
||||
y5: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
||||
Detailed RabbitMQ CPU consumption
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes CPU consumption during RPC CALL load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master total, %
|
||||
y1: Slave 1 total, %
|
||||
y2: Slave 2 total, %
|
||||
z0: Master sys, %
|
||||
z1: Slave 1 sys, %
|
||||
z2: Slave 2 sys, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: call }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
rabbit_sys_0: { $avg: "$rabbit_sys_0" }
|
||||
rabbit_sys_1: { $avg: "$rabbit_sys_1" }
|
||||
rabbit_sys_2: { $avg: "$rabbit_sys_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $multiply: [ "$rabbit_total_0", 100 ] }
|
||||
y1: { $multiply: [ "$rabbit_total_1", 100 ] }
|
||||
y2: { $multiply: [ "$rabbit_total_2", 100 ] }
|
||||
z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
|
||||
z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
|
||||
z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ resident memory
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes memory consumption during RPC CALL load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master, Mb
|
||||
y1: Slave 1, Mb
|
||||
y2: Slave 2, Mb
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: call }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
|
@ -81,11 +152,13 @@ depending on number of concurrent threads.
|
|||
Test Case 2: RPC CAST Throughput Test
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
**Message processing**
|
||||
Message processing
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Messages are collected at 2 points: ``sent`` - messages sent by the client
|
||||
and ``received`` - messages received by the server. Also the number of lost
|
||||
messages is calculated.
|
||||
messages is calculated. Sizes of messages is based on the distribution of
|
||||
messages collected on the 100-node cloud.
|
||||
|
||||
{{'''
|
||||
title: RPC CAST Message count
|
||||
|
|
@ -97,7 +170,7 @@ messages is calculated.
|
|||
chart: line
|
||||
pipeline:
|
||||
- { $match: { task: omsimulator, mode: cast }}
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
|
||||
sent: { $sum: "$client.count" },
|
||||
received: { $sum: "$server.count" },
|
||||
lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
|
||||
|
|
@ -112,7 +185,8 @@ messages is calculated.
|
|||
}}
|
||||
|
||||
|
||||
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
|
||||
The throughput, latency and RabbitMQ CPU utilization
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
|
||||
depending on number of concurrent threads.
|
||||
|
|
@ -127,20 +201,86 @@ depending on number of concurrent threads.
|
|||
y4: RabbitMQ CPU consumption, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- { $match: { task: omsimulator, mode: cast }}
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
|
||||
latency: { $avg: "$server.latency" },
|
||||
rabbit_total: { $avg: "$rabbit_total" }
|
||||
}}
|
||||
- { $project: { x: "$_id.threads",
|
||||
y: "$msg_sent_per_sec",
|
||||
y2: "$msg_received_per_sec",
|
||||
y3: { $multiply: [ "$latency", 1000 ] },
|
||||
y4: { $multiply: [ "$rabbit_total", 100 ] }
|
||||
}}
|
||||
- { $sort: { x: 1 }}
|
||||
- $match: { task: omsimulator, mode: cast }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
||||
latency: { $avg: "$server.latency" }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y: "$msg_sent_per_sec"
|
||||
y2: "$msg_received_per_sec"
|
||||
y3: { $multiply: [ "$latency", 1000 ] }
|
||||
y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ CPU consumption
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes CPU consumption during RPC CAST load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master total, %
|
||||
y1: Slave 1 total, %
|
||||
y2: Slave 2 total, %
|
||||
z0: Master sys, %
|
||||
z1: Slave 1 sys, %
|
||||
z2: Slave 2 sys, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: cast }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
rabbit_sys_0: { $avg: "$rabbit_sys_0" }
|
||||
rabbit_sys_1: { $avg: "$rabbit_sys_1" }
|
||||
rabbit_sys_2: { $avg: "$rabbit_sys_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $multiply: [ "$rabbit_total_0", 100 ] }
|
||||
y1: { $multiply: [ "$rabbit_total_1", 100 ] }
|
||||
y2: { $multiply: [ "$rabbit_total_2", 100 ] }
|
||||
z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
|
||||
z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
|
||||
z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ resident memory
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes memory consumption during RPC CAST load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master, Mb
|
||||
y1: Slave 1, Mb
|
||||
y2: Slave 2, Mb
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: cast }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
|
@ -148,11 +288,13 @@ depending on number of concurrent threads.
|
|||
Test Case 3: Notification Throughput Test
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
**Message processing**
|
||||
Message processing
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Messages are collected at 2 points: ``sent`` - messages sent by the client
|
||||
and ``received`` - messages received by the server. Also the number of lost
|
||||
messages is calculated.
|
||||
messages is calculated. Sizes of messages is based on the distribution of
|
||||
messages collected on the 100-node cloud.
|
||||
|
||||
{{'''
|
||||
title: NOTIFY Message count
|
||||
|
|
@ -164,7 +306,7 @@ messages is calculated.
|
|||
chart: line
|
||||
pipeline:
|
||||
- { $match: { task: omsimulator, mode: notify }}
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
|
||||
sent: { $sum: "$client.count" },
|
||||
received: { $sum: "$server.count" },
|
||||
lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
|
||||
|
|
@ -179,7 +321,8 @@ messages is calculated.
|
|||
}}
|
||||
|
||||
|
||||
**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
|
||||
The throughput, latency and RabbitMQ CPU utilization
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The chart shows the throughput, latency and CPU utilization by RabbitMQ server
|
||||
depending on number of concurrent threads.
|
||||
|
|
@ -194,23 +337,90 @@ depending on number of concurrent threads.
|
|||
y4: RabbitMQ CPU consumption, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- { $match: { task: omsimulator, mode: notify }}
|
||||
- { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
|
||||
latency: { $avg: "$server.latency" },
|
||||
rabbit_total: { $avg: "$rabbit_total" }
|
||||
}}
|
||||
- { $project: { x: "$_id.threads",
|
||||
y: "$msg_sent_per_sec",
|
||||
y2: "$msg_received_per_sec",
|
||||
y3: { $multiply: [ "$latency", 1000 ] },
|
||||
y4: { $multiply: [ "$rabbit_total", 100 ] }
|
||||
}}
|
||||
- { $sort: { x: 1 }}
|
||||
- $match: { task: omsimulator, mode: notify }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
||||
latency: { $avg: "$server.latency" }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y: "$msg_sent_per_sec"
|
||||
y2: "$msg_received_per_sec"
|
||||
y3: { $multiply: [ "$latency", 1000 ] }
|
||||
y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ CPU consumption
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes CPU consumption during NOTIFY load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master total, %
|
||||
y1: Slave 1 total, %
|
||||
y2: Slave 2 total, %
|
||||
z0: Master sys, %
|
||||
z1: Slave 1 sys, %
|
||||
z2: Slave 2 sys, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: notify }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
rabbit_sys_0: { $avg: "$rabbit_sys_0" }
|
||||
rabbit_sys_1: { $avg: "$rabbit_sys_1" }
|
||||
rabbit_sys_2: { $avg: "$rabbit_sys_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $multiply: [ "$rabbit_total_0", 100 ] }
|
||||
y1: { $multiply: [ "$rabbit_total_1", 100 ] }
|
||||
y2: { $multiply: [ "$rabbit_total_2", 100 ] }
|
||||
z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
|
||||
z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
|
||||
z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ resident memory
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes memory consumption during NOTIFY load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master, Mb
|
||||
y1: Slave 1, Mb
|
||||
y2: Slave 2, Mb
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: notify }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
||||
.. references:
|
||||
|
||||
.. _message_queue_performance: http://docs.openstack.org/developer/performance-docs/test_plans/mq/plan.html
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ execution:
|
|||
-
|
||||
hosts: {{ tester_hosts }}
|
||||
matrix:
|
||||
host_count: [ 1, 2, 5, 10 ]
|
||||
processes: [ 1, 2, 5, 7, 10 ]
|
||||
tasks:
|
||||
- omsimulator:
|
||||
mode: call
|
||||
|
|
@ -61,7 +61,7 @@ execution:
|
|||
-
|
||||
hosts: {{ tester_hosts }}
|
||||
matrix:
|
||||
host_count: [ 1, 2, 5, 10 ]
|
||||
processes: [ 1, 2, 5, 7, 10 ]
|
||||
tasks:
|
||||
- omsimulator:
|
||||
mode: cast
|
||||
|
|
@ -73,7 +73,7 @@ execution:
|
|||
-
|
||||
hosts: {{ tester_hosts }}
|
||||
matrix:
|
||||
host_count: [ 1, 2, 5, 10 ]
|
||||
processes: [ 1, 2, 5, 7, 10 ]
|
||||
tasks:
|
||||
- omsimulator:
|
||||
mode: notify
|
||||
|
|
@ -87,7 +87,7 @@ execution:
|
|||
tasks:
|
||||
- atop:
|
||||
command: stop
|
||||
labels: [ PRC ]
|
||||
labels: [ PRC, PRM ]
|
||||
|
||||
aggregation:
|
||||
-
|
||||
|
|
@ -96,8 +96,54 @@ aggregation:
|
|||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
|
||||
- { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
|
||||
{% if rabbit_hosts[1] %}
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
{% if rabbit_hosts[2] %}
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
report:
|
||||
template: omsimulator.rst
|
||||
|
|
|
|||
Loading…
Reference in New Issue