diff --git a/performa/modules/omsimulator.py b/performa/modules/omsimulator.py index 2650e11..bc8ef73 100644 --- a/performa/modules/omsimulator.py +++ b/performa/modules/omsimulator.py @@ -1,5 +1,6 @@ #!/usr/bin/python import copy +import multiprocessing import os import signal import tempfile @@ -11,35 +12,6 @@ UNIQUE_NAME = 'performa_omsimulator' DIR = '/tmp/performa/oslo.messaging/tools/' PYTHON = '/tmp/performa/oslo.messaging/.venv/bin/python' -PATTERNS = [ - r'(?P\d+) messages were sent for (?P\d+) sec', - r'(?P\d+) bytes were sent for', -] -TRANSFORM_FIELDS = { - 'msg_sent': int, - 'bytes_sent': int, - 'duration': int, -} - - -def parse_output(raw): - result = {} - - for pattern in PATTERNS: - for parsed in re.finditer(pattern, raw): - result.update(parsed.groupdict()) - - for k in result.keys(): - if k in TRANSFORM_FIELDS: - result[k] = TRANSFORM_FIELDS[k](result[k]) - - result['msg_sent_bandwidth'] = (result.get('msg_sent', 0) / - result.get('duration', 1)) - result['bytes_sent_bandwidth'] = (result.get('bytes_sent', 0) / - result.get('duration', 1)) - - return result - def chdir(module): try: @@ -48,29 +20,6 @@ def chdir(module): module.fail_json(msg='Failed to change dir to %s: %s' % (DIR, e)) -def start_daemon(module, cmd): - cmd = ('daemon -n %(name)s -D %(dir)s -F %(pid)s -U -- %(cmd)s' % - dict(name=UNIQUE_NAME, dir=DIR, pid=SERVER_PID, cmd=cmd)) - - rc, stdout, stderr = module.run_command(cmd) - result = dict(changed=True, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd) - - if rc: - module.fail_json(msg='Failed to start omsimulator', **result) - - -def stop_daemon(module): - rc, stdout, stderr = module.run_command('/bin/cat %s' % SERVER_PID) - - if rc: - return - - rc, stdout, stderr = module.run_command('pgrep -P %s' % stdout) - os.kill(int(stdout), signal.SIGINT) - - time.sleep(2) - - def read_file(filename): fd = None try: @@ -92,35 +41,16 @@ def transform_series(series): return result -def run(module): - params = copy.deepcopy(module.params) +def make_file_name(base, index): + return '%s-%s' % (base, index) - if params['mode'] == 'notify': - server_tool = 'notify-server' - client_tool = 'notify-client' - else: - server_tool = 'rpc-server' - client_tool = 'rpc-client' - - params['python'] = PYTHON - # todo: fix topic support in omsimulator - # params['topic'] = 'performa-%d' % (random.random() * 1000000) - params['server_tool'] = server_tool - params['client_tool'] = client_tool - params['server_file'] = SERVER_FILE_NAME - params['client_file'] = CLIENT_FILE_NAME - - params['url'] = params['server_url'] or params['url'] - server = ('%(python)s simulator.py ' - # '--topic %(topic)s ' - '--url %(url)s ' - '--json %(server_file)s ' - '%(server_tool)s ') % params +def make_client_cmd(params, i): + params['client_file'] = make_file_name(CLIENT_FILE_NAME, i) params['url'] = params['client_url'] or params['url'] + client = ('%(python)s simulator.py ' - # '--topic %(topic)s ' - '--url=%(url)s ' + '--url %(url)s ' '--json %(client_file)s ' '-l %(duration)s ' '%(client_tool)s ' @@ -134,45 +64,121 @@ def run(module): if params['mode'] == 'fanout': client += '--is-fanout True ' - start_daemon(module, server) + return client - rc, stdout, stderr = module.run_command(client) - if rc: - module.fail_json(msg='Failed to start omsimulator', - stderr=stderr, rc=rc, cmd=client) +def make_server_cmd(params, i): + params['server_file'] = make_file_name(SERVER_FILE_NAME, i) + params['url'] = params['server_url'] or params['url'] - stop_daemon(module) + server = ('%(python)s simulator.py ' + '--url %(url)s ' + '--json %(server_file)s ' + '%(server_tool)s ') % params - try: - client_data = read_file(CLIENT_FILE_NAME) - server_data = read_file(SERVER_FILE_NAME) + return server + + +def run_client(module, command): + module.run_command(command) + + +def run_client_pool(module, params): + processes = [] + + for i in range(params['processes']): + cmd = make_client_cmd(params, i) + p = multiprocessing.Process(target=run_client, args=(module, cmd)) + processes.append(p) + p.start() + + for p in processes: + p.join() + + +def run_server(module, command): + module.run_command(command) + + +def start_server_pool(module, params): + processes = [] + + for i in range(params['processes']): + cmd = make_server_cmd(params, i) + p = multiprocessing.Process(target=run_client, args=(module, cmd)) + processes.append(p) + p.start() + + return processes + + +def stop_server_pool(module, processes): + for p in processes: + rc, stdout, stderr = module.run_command('pgrep -P %s' % p.pid) + + for child in (int(p) for p in stdout.split('\n') if p): + os.kill(child, signal.SIGINT) + + time.sleep(3) # let simulators handle the signal + + for p in processes: + os.kill(p.pid, signal.SIGINT) + p.join() + + +def collect_results(params): + result = dict(records=[], series=[]) + + for i in range(params['processes']): + client_data = read_file(make_file_name(CLIENT_FILE_NAME, i)) + server_data = read_file(make_file_name(SERVER_FILE_NAME, i)) client_summary = client_data['summary']['client'] - - record = dict(start=client_summary['start'], - end=client_summary['end'], + record = dict(start=client_summary['start'], end=client_summary['end'], client=client_summary) - if 'round_trip' in client_data['summary']: round_trip_summary = client_data['summary']['round_trip'] record['round_trip'] = round_trip_summary - if 'error' in client_data['summary']: error_summary = client_data['summary']['error'] record['error'] = error_summary - server_summary = server_data['summary'] record['server'] = server_summary - series = transform_series(client_data['series']) series.extend(transform_series(server_data['series'])) - result = dict(records=[record], series=series) + result['records'].append(record) + result['series'] += series + + return result + + +def run(module): + params = copy.deepcopy(module.params) + + if params['mode'] == 'notify': + server_tool = 'notify-server' + client_tool = 'notify-client' + else: + server_tool = 'rpc-server' + client_tool = 'rpc-client' + + params['python'] = PYTHON + params['server_tool'] = server_tool + params['client_tool'] = client_tool + + server_processes = start_server_pool(module, params) + + run_client_pool(module, params) + + stop_server_pool(module, server_processes) + + try: + result = collect_results(params) module.exit_json(**result) except Exception as e: msg = 'Failed to read omsimulator output: %s' % e - module.fail_json(msg=msg, rc=rc, stderr=stderr, stdout=stdout) + module.fail_json(msg=msg) def main(): @@ -184,6 +190,7 @@ def main(): client_url=dict(), server_url=dict(), threads=dict(type='int', default=10), + processes=dict(type='int', default=1), duration=dict(type='int', default=10), timeout=dict(type='int', default=5), sending_delay=dict(type='float', default=-1.0), diff --git a/performa/scenarios/mq/omsimulator-threading.yaml b/performa/scenarios/mq/omsimulator-threading.yaml index 16d091e..4949ddd 100644 --- a/performa/scenarios/mq/omsimulator-threading.yaml +++ b/performa/scenarios/mq/omsimulator-threading.yaml @@ -84,7 +84,7 @@ execution: tasks: - atop: command: stop - labels: [ PRC ] + labels: [ PRC, PRM ] aggregation: - @@ -93,8 +93,54 @@ aggregation: { task: omsimulator } values: pipeline: - - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }} - - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }} + - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} } + - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} } + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} } + - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } } + +{% if rabbit_hosts[1] %} + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} } + - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} } + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} } + - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } } +{% endif %} + +{% if rabbit_hosts[2] %} + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} } + - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} } + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} } + - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } } +{% endif %} report: template: omsimulator.rst diff --git a/performa/scenarios/mq/omsimulator.rst b/performa/scenarios/mq/omsimulator.rst index 55dafbf..5aff3cb 100644 --- a/performa/scenarios/mq/omsimulator.rst +++ b/performa/scenarios/mq/omsimulator.rst @@ -8,11 +8,14 @@ with `Oslo.messaging Simulator`_ Test Case 1: RPC CALL Throughput Test ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Message processing** +Message processing +~~~~~~~~~~~~~~~~~~ Messages are collected at 3 points: ``sent`` - messages sent by the client, ``received`` - messages received by the server, ``round-trip`` - replies received by the client. Also the number of lost messages is calculated. +Sizes of messages is based on the distribution of messages collected on +the 100-node cloud. {{''' title: RPC CALL Message count @@ -25,7 +28,7 @@ received by the client. Also the number of lost messages is calculated. chart: line pipeline: - { $match: { task: omsimulator, mode: call }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, + - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }, sent: { $sum: "$client.count" }, received: { $sum: "$server.count" }, round_trip: { $sum: "$round_trip.count" }, @@ -42,7 +45,8 @@ received by the client. Also the number of lost messages is calculated. }} -**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** +The throughput, latency and RabbitMQ CPU utilization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The chart shows the throughput, latency and CPU utilization by RabbitMQ server depending on number of concurrent threads. @@ -58,22 +62,89 @@ depending on number of concurrent threads. y5: RabbitMQ CPU consumption, % chart: line pipeline: - - { $match: { task: omsimulator, mode: call }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}, - msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}, - msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}, - latency: { $avg: "$round_trip.latency" }, - rabbit_total: { $avg: "$rabbit_total" } - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: "$msg_received_per_sec", - y3: "$msg_round_trip_per_sec", - y4: { $multiply: [ "$latency", 1000 ] }, - y5: { $multiply: [ "$rabbit_total", 100 ] } - }} - - { $sort: { x: 1 }} + - $match: { task: omsimulator, mode: call } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }} + msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }} + msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }} + latency: { $avg: "$round_trip.latency" } + rabbit_total_0: { $avg: "$rabbit_total_0" } + rabbit_total_1: { $avg: "$rabbit_total_1" } + rabbit_total_2: { $avg: "$rabbit_total_2" } + - $project: + x: "$_id.threads" + y: "$msg_sent_per_sec" + y2: "$msg_received_per_sec" + y3: "$msg_round_trip_per_sec" + y4: { $multiply: [ "$latency", 1000 ] } + y5: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] } +''' | chart_and_table +}} + + +Detailed RabbitMQ CPU consumption +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thus chart shows statistics on RabbitMQ CPU consumption per nodes. + +{{''' + title: RabbitMQ nodes CPU consumption during RPC CALL load test + axes: + x: threads + y0: Master total, % + y1: Slave 1 total, % + y2: Slave 2 total, % + z0: Master sys, % + z1: Slave 1 sys, % + z2: Slave 2 sys, % + chart: line + pipeline: + - $match: { task: omsimulator, mode: call } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + rabbit_total_0: { $avg: "$rabbit_total_0" } + rabbit_total_1: { $avg: "$rabbit_total_1" } + rabbit_total_2: { $avg: "$rabbit_total_2" } + rabbit_sys_0: { $avg: "$rabbit_sys_0" } + rabbit_sys_1: { $avg: "$rabbit_sys_1" } + rabbit_sys_2: { $avg: "$rabbit_sys_2" } + - $project: + x: "$_id.threads" + y0: { $multiply: [ "$rabbit_total_0", 100 ] } + y1: { $multiply: [ "$rabbit_total_1", 100 ] } + y2: { $multiply: [ "$rabbit_total_2", 100 ] } + z0: { $multiply: [ "$rabbit_sys_0", 100 ] } + z1: { $multiply: [ "$rabbit_sys_1", 100 ] } + z2: { $multiply: [ "$rabbit_sys_2", 100 ] } +''' | chart_and_table +}} + +Detailed RabbitMQ resident memory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes. + +{{''' + title: RabbitMQ nodes memory consumption during RPC CALL load test + axes: + x: threads + y0: Master, Mb + y1: Slave 1, Mb + y2: Slave 2, Mb + chart: line + pipeline: + - $match: { task: omsimulator, mode: call } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + rabbit_0: { $avg: "$rabbit_resident_0" } + rabbit_1: { $avg: "$rabbit_resident_1" } + rabbit_2: { $avg: "$rabbit_resident_2" } + - $project: + x: "$_id.threads" + y0: { $divide: [ "$rabbit_0", 1048576 ] } + y1: { $divide: [ "$rabbit_1", 1048576 ] } + y2: { $divide: [ "$rabbit_2", 1048576 ] } ''' | chart_and_table }} @@ -81,11 +152,13 @@ depending on number of concurrent threads. Test Case 2: RPC CAST Throughput Test ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Message processing** +Message processing +~~~~~~~~~~~~~~~~~~ Messages are collected at 2 points: ``sent`` - messages sent by the client and ``received`` - messages received by the server. Also the number of lost -messages is calculated. +messages is calculated. Sizes of messages is based on the distribution of +messages collected on the 100-node cloud. {{''' title: RPC CAST Message count @@ -97,7 +170,7 @@ messages is calculated. chart: line pipeline: - { $match: { task: omsimulator, mode: cast }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, + - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }, sent: { $sum: "$client.count" }, received: { $sum: "$server.count" }, lost: { $sum: { $subtract: ["$client.count", "$server.count"] }} @@ -112,7 +185,8 @@ messages is calculated. }} -**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** +The throughput, latency and RabbitMQ CPU utilization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The chart shows the throughput, latency and CPU utilization by RabbitMQ server depending on number of concurrent threads. @@ -127,20 +201,86 @@ depending on number of concurrent threads. y4: RabbitMQ CPU consumption, % chart: line pipeline: - - { $match: { task: omsimulator, mode: cast }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}, - msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}, - latency: { $avg: "$server.latency" }, - rabbit_total: { $avg: "$rabbit_total" } - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: "$msg_received_per_sec", - y3: { $multiply: [ "$latency", 1000 ] }, - y4: { $multiply: [ "$rabbit_total", 100 ] } - }} - - { $sort: { x: 1 }} + - $match: { task: omsimulator, mode: cast } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }} + msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }} + latency: { $avg: "$server.latency" } + rabbit_total_0: { $avg: "$rabbit_total_0" } + rabbit_total_1: { $avg: "$rabbit_total_1" } + rabbit_total_2: { $avg: "$rabbit_total_2" } + - $project: + x: "$_id.threads" + y: "$msg_sent_per_sec" + y2: "$msg_received_per_sec" + y3: { $multiply: [ "$latency", 1000 ] } + y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] } +''' | chart_and_table +}} + +Detailed RabbitMQ CPU consumption +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thus chart shows statistics on RabbitMQ CPU consumption per nodes. + +{{''' + title: RabbitMQ nodes CPU consumption during RPC CAST load test + axes: + x: threads + y0: Master total, % + y1: Slave 1 total, % + y2: Slave 2 total, % + z0: Master sys, % + z1: Slave 1 sys, % + z2: Slave 2 sys, % + chart: line + pipeline: + - $match: { task: omsimulator, mode: cast } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + rabbit_total_0: { $avg: "$rabbit_total_0" } + rabbit_total_1: { $avg: "$rabbit_total_1" } + rabbit_total_2: { $avg: "$rabbit_total_2" } + rabbit_sys_0: { $avg: "$rabbit_sys_0" } + rabbit_sys_1: { $avg: "$rabbit_sys_1" } + rabbit_sys_2: { $avg: "$rabbit_sys_2" } + - $project: + x: "$_id.threads" + y0: { $multiply: [ "$rabbit_total_0", 100 ] } + y1: { $multiply: [ "$rabbit_total_1", 100 ] } + y2: { $multiply: [ "$rabbit_total_2", 100 ] } + z0: { $multiply: [ "$rabbit_sys_0", 100 ] } + z1: { $multiply: [ "$rabbit_sys_1", 100 ] } + z2: { $multiply: [ "$rabbit_sys_2", 100 ] } +''' | chart_and_table +}} + +Detailed RabbitMQ resident memory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes. + +{{''' + title: RabbitMQ nodes memory consumption during RPC CAST load test + axes: + x: threads + y0: Master, Mb + y1: Slave 1, Mb + y2: Slave 2, Mb + chart: line + pipeline: + - $match: { task: omsimulator, mode: cast } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + rabbit_0: { $avg: "$rabbit_resident_0" } + rabbit_1: { $avg: "$rabbit_resident_1" } + rabbit_2: { $avg: "$rabbit_resident_2" } + - $project: + x: "$_id.threads" + y0: { $divide: [ "$rabbit_0", 1048576 ] } + y1: { $divide: [ "$rabbit_1", 1048576 ] } + y2: { $divide: [ "$rabbit_2", 1048576 ] } ''' | chart_and_table }} @@ -148,11 +288,13 @@ depending on number of concurrent threads. Test Case 3: Notification Throughput Test ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Message processing** +Message processing +~~~~~~~~~~~~~~~~~~ Messages are collected at 2 points: ``sent`` - messages sent by the client and ``received`` - messages received by the server. Also the number of lost -messages is calculated. +messages is calculated. Sizes of messages is based on the distribution of +messages collected on the 100-node cloud. {{''' title: NOTIFY Message count @@ -164,7 +306,7 @@ messages is calculated. chart: line pipeline: - { $match: { task: omsimulator, mode: notify }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, + - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }, sent: { $sum: "$client.count" }, received: { $sum: "$server.count" }, lost: { $sum: { $subtract: ["$client.count", "$server.count"] }} @@ -179,7 +321,8 @@ messages is calculated. }} -**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** +The throughput, latency and RabbitMQ CPU utilization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The chart shows the throughput, latency and CPU utilization by RabbitMQ server depending on number of concurrent threads. @@ -194,23 +337,90 @@ depending on number of concurrent threads. y4: RabbitMQ CPU consumption, % chart: line pipeline: - - { $match: { task: omsimulator, mode: notify }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}, - msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}, - latency: { $avg: "$server.latency" }, - rabbit_total: { $avg: "$rabbit_total" } - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: "$msg_received_per_sec", - y3: { $multiply: [ "$latency", 1000 ] }, - y4: { $multiply: [ "$rabbit_total", 100 ] } - }} - - { $sort: { x: 1 }} + - $match: { task: omsimulator, mode: notify } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }} + msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }} + latency: { $avg: "$server.latency" } + rabbit_total_0: { $avg: "$rabbit_total_0" } + rabbit_total_1: { $avg: "$rabbit_total_1" } + rabbit_total_2: { $avg: "$rabbit_total_2" } + - $project: + x: "$_id.threads" + y: "$msg_sent_per_sec" + y2: "$msg_received_per_sec" + y3: { $multiply: [ "$latency", 1000 ] } + y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] } ''' | chart_and_table }} +Detailed RabbitMQ CPU consumption +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thus chart shows statistics on RabbitMQ CPU consumption per nodes. + +{{''' + title: RabbitMQ nodes CPU consumption during NOTIFY load test + axes: + x: threads + y0: Master total, % + y1: Slave 1 total, % + y2: Slave 2 total, % + z0: Master sys, % + z1: Slave 1 sys, % + z2: Slave 2 sys, % + chart: line + pipeline: + - $match: { task: omsimulator, mode: notify } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + rabbit_total_0: { $avg: "$rabbit_total_0" } + rabbit_total_1: { $avg: "$rabbit_total_1" } + rabbit_total_2: { $avg: "$rabbit_total_2" } + rabbit_sys_0: { $avg: "$rabbit_sys_0" } + rabbit_sys_1: { $avg: "$rabbit_sys_1" } + rabbit_sys_2: { $avg: "$rabbit_sys_2" } + - $project: + x: "$_id.threads" + y0: { $multiply: [ "$rabbit_total_0", 100 ] } + y1: { $multiply: [ "$rabbit_total_1", 100 ] } + y2: { $multiply: [ "$rabbit_total_2", 100 ] } + z0: { $multiply: [ "$rabbit_sys_0", 100 ] } + z1: { $multiply: [ "$rabbit_sys_1", 100 ] } + z2: { $multiply: [ "$rabbit_sys_2", 100 ] } +''' | chart_and_table +}} + +Detailed RabbitMQ resident memory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes. + +{{''' + title: RabbitMQ nodes memory consumption during NOTIFY load test + axes: + x: threads + y0: Master, Mb + y1: Slave 1, Mb + y2: Slave 2, Mb + chart: line + pipeline: + - $match: { task: omsimulator, mode: notify } + - $group: + _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } } + rabbit_0: { $avg: "$rabbit_resident_0" } + rabbit_1: { $avg: "$rabbit_resident_1" } + rabbit_2: { $avg: "$rabbit_resident_2" } + - $project: + x: "$_id.threads" + y0: { $divide: [ "$rabbit_0", 1048576 ] } + y1: { $divide: [ "$rabbit_1", 1048576 ] } + y2: { $divide: [ "$rabbit_2", 1048576 ] } +''' | chart_and_table +}} + + .. references: .. _message_queue_performance: http://docs.openstack.org/developer/performance-docs/test_plans/mq/plan.html diff --git a/performa/scenarios/mq/omsimulator.yaml b/performa/scenarios/mq/omsimulator.yaml index f5b4267..7c30d39 100644 --- a/performa/scenarios/mq/omsimulator.yaml +++ b/performa/scenarios/mq/omsimulator.yaml @@ -49,7 +49,7 @@ execution: - hosts: {{ tester_hosts }} matrix: - host_count: [ 1, 2, 5, 10 ] + processes: [ 1, 2, 5, 7, 10 ] tasks: - omsimulator: mode: call @@ -61,7 +61,7 @@ execution: - hosts: {{ tester_hosts }} matrix: - host_count: [ 1, 2, 5, 10 ] + processes: [ 1, 2, 5, 7, 10 ] tasks: - omsimulator: mode: cast @@ -73,7 +73,7 @@ execution: - hosts: {{ tester_hosts }} matrix: - host_count: [ 1, 2, 5, 10 ] + processes: [ 1, 2, 5, 7, 10 ] tasks: - omsimulator: mode: notify @@ -87,7 +87,7 @@ execution: tasks: - atop: command: stop - labels: [ PRC ] + labels: [ PRC, PRM ] aggregation: - @@ -96,8 +96,54 @@ aggregation: { task: omsimulator } values: pipeline: - - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }} - - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }} + - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} } + - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} } + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} } + - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } } + +{% if rabbit_hosts[1] %} + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} } + - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} } + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} } + - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } } +{% endif %} + +{% if rabbit_hosts[2] %} + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} } + - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} } + - + update: + query: + { task: omsimulator } + values: + pipeline: + - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} } + - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } } +{% endif %} report: template: omsimulator.rst