summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIlya Shakhat <ishakhat@mirantis.com>2016-03-22 16:37:50 +0300
committerIlya Shakhat <ishakhat@mirantis.com>2016-03-22 16:37:50 +0300
commit225fefe4db33f5a12b487ff1d9d368460f02cec9 (patch)
tree22d2f7308afcde5b217f4d2da9da43d72ccdbff7
parent85a252859d635bed9c5c6d4cea218686653ef474 (diff)
Polish MQ HA scenarios and reports
-rw-r--r--performa/engine/report.py4
-rw-r--r--performa/modules/omsimulator.py8
-rw-r--r--performa/scenarios/mq/omsimulator-ha-call.rst109
-rw-r--r--performa/scenarios/mq/omsimulator-ha-cast.rst91
-rw-r--r--performa/scenarios/mq/omsimulator-ha-notify.rst91
5 files changed, 214 insertions, 89 deletions
diff --git a/performa/engine/report.py b/performa/engine/report.py
index 52fe699..b0ee970 100644
--- a/performa/engine/report.py
+++ b/performa/engine/report.py
@@ -158,8 +158,10 @@ def generate_info(definition_str, db, doc_folder, tag):
158 158
159 for field_name, field_title in sorted(fields.items(), key=lambda a: a[0]): 159 for field_name, field_title in sorted(fields.items(), key=lambda a: a[0]):
160 value = data[field_name] 160 value = data[field_name]
161 if value is None:
162 value = '.'
161 patt = (''' *\n - %%s\n - %%%s\n''' % 163 patt = (''' *\n - %%s\n - %%%s\n''' %
162 ('d' if isinstance(value, int) else '.1f')) 164 ('.1f' if isinstance(value, float) else 's'))
163 table += patt % (field_title, value) 165 table += patt % (field_title, value)
164 166
165 return table 167 return table
diff --git a/performa/modules/omsimulator.py b/performa/modules/omsimulator.py
index 49647e0..07a12e7 100644
--- a/performa/modules/omsimulator.py
+++ b/performa/modules/omsimulator.py
@@ -121,6 +121,8 @@ def run(module):
121 '--json %(client_file)s ' 121 '--json %(client_file)s '
122 '-l %(duration)s ' 122 '-l %(duration)s '
123 '%(client_tool)s ' 123 '%(client_tool)s '
124 '--timeout %(timeout)s '
125 '-w %(sending_delay)s '
124 '-p %(threads)s ') % params 126 '-p %(threads)s ') % params
125 127
126 if params['mode'] == 'cast': 128 if params['mode'] == 'cast':
@@ -153,6 +155,10 @@ def run(module):
153 round_trip_summary = client_data['summary']['round_trip'] 155 round_trip_summary = client_data['summary']['round_trip']
154 record['round_trip'] = round_trip_summary 156 record['round_trip'] = round_trip_summary
155 157
158 if 'error' in client_data['summary']:
159 error_summary = client_data['summary']['error']
160 record['error'] = error_summary
161
156 server_summary = server_data['summary'] 162 server_summary = server_data['summary']
157 record['server'] = server_summary 163 record['server'] = server_summary
158 164
@@ -174,6 +180,8 @@ def main():
174 url=dict(required=True), 180 url=dict(required=True),
175 threads=dict(type='int', default=10), 181 threads=dict(type='int', default=10),
176 duration=dict(type='int', default=10), 182 duration=dict(type='int', default=10),
183 timeout=dict(type='int', default=5),
184 sending_delay=dict(type='float', default=-1.0),
177 )) 185 ))
178 186
179 chdir(module) 187 chdir(module)
diff --git a/performa/scenarios/mq/omsimulator-ha-call.rst b/performa/scenarios/mq/omsimulator-ha-call.rst
index df0ff43..c4027c8 100644
--- a/performa/scenarios/mq/omsimulator-ha-call.rst
+++ b/performa/scenarios/mq/omsimulator-ha-call.rst
@@ -1,43 +1,58 @@
1Oslo.messaging simulator HA report 1RPC CALL fail-over test report
2---------------------------------- 2------------------------------
3 3
4This report is result of `message_queue_performance`_ execution 4This scenario is executed with help of oslo.messaging simulator. There is
5with `Oslo.messaging Simulator`_ 5one client-server pair of simulator running in single-threaded mode. The
6stats are collected from both client and server and detailed report is shown
7with one second precision.
6 8
7 9
8RPC CALL fail-over throughput test 10Execution Summary
9^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 11^^^^^^^^^^^^^^^^^
10
11**Summary**
12 12
13{{''' 13{{'''
14 title: Execution summary 14 title: RPC CALL Execution summary
15 fields: 15 fields:
16 a1: Client sent, msg 16 a1: Client sent, msg
17 b1: Server received, msg 17 a2: Server received, msg
18 b2: Client received replies, msg 18 a3: Client received replies, msg
19 b3: Loss, msg 19 a4: Loss, msg
20 c1: Avg. request latency, ms 20 a5: Errors, msg
21 c2: Max request latency, ms 21 b1: Duration, sec
22 c3: Avg. round-trip latency, ms 22 c1: Throughput, msg/sec
23 c4: Max round-trip latency, ms 23 d1: Transfer, Mb
24 d2: Bandwidth, Mb/sec
25 e1: Avg. request latency, ms
26 e2: Max request latency, ms
27 e3: Avg. round-trip latency, ms
28 e4: Max round-trip latency, ms
24 collection: records 29 collection: records
25 pipeline: 30 pipeline:
26 - $match: { task: omsimulator, mode: call } 31 - $match: { task: omsimulator, mode: call }
27 - $project: 32 - $project:
28 a1: "$client.count" 33 a1: "$client.count"
29 b1: "$server.count" 34 a2: "$server.count"
30 b2: "$round_trip.count" 35 a3: "$round_trip.count"
31 b3: { $subtract: ["$client.count", "$round_trip.count" ] } 36 a4: { $subtract: ["$client.count", "$round_trip.count" ] }
32 c1: { $multiply: ["$server.latency", 1000] } 37 a5: "$error.count"
33 c2: { $multiply: ["$server.max_latency", 1000] } 38 b1: "$client.duration"
34 c3: { $multiply: ["$round_trip.latency", 1000] } 39 c1: { $divide: ["$client.count", "$client.duration"] }
35 c4: { $multiply: ["$round_trip.max_latency", 1000] } 40 d1: { $divide: ["$client.size", 1048576] }
41 d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
42 e1: { $multiply: ["$server.latency", 1000] }
43 e2: { $multiply: ["$server.max_latency", 1000] }
44 e3: { $multiply: ["$round_trip.latency", 1000] }
45 e4: { $multiply: ["$round_trip.max_latency", 1000] }
36''' | info 46''' | info
37}} 47}}
38 48
39 49
40**Message flow** 50Message flow
51^^^^^^^^^^^^
52
53This chart shows the message flow between client and server. It includes
54messages sent by the client, received by the server, replies received by
55the client and errors caught by the client.
41 56
42{{''' 57{{'''
43 title: RPC CALL message flow 58 title: RPC CALL message flow
@@ -46,7 +61,7 @@ RPC CALL fail-over throughput test
46 y1: sent, msg 61 y1: sent, msg
47 y2: received, msg 62 y2: received, msg
48 y3: round-trip, msg 63 y3: round-trip, msg
49 y4: latency, ms 64 y4: errors, msg
50 chart: line 65 chart: line
51 collection: series 66 collection: series
52 pipelines: 67 pipelines:
@@ -65,29 +80,52 @@ RPC CALL fail-over throughput test
65 - $project: 80 - $project:
66 x: "$timestamp" 81 x: "$timestamp"
67 y3: "$count" 82 y3: "$count"
68 y4: { $multiply: ["$latency", 1000] } 83 -
84 - $match: { task: omsimulator, mode: call, name: error_0 }
85 - $project:
86 x: "$timestamp"
87 y4: "$count"
69''' | chart 88''' | chart
70}} 89}}
71 90
91where:
92 * ``sent`` - messages sent by the client
93 * ``received`` - messages received by the server
94 * ``round-trip`` - replies to messages received by the client
95 * ``errors`` - errors exposed and caught by the client
72 96
73**Messages sent by the client** 97
98Messages sent by the client
99^^^^^^^^^^^^^^^^^^^^^^^^^^^
100
101This chart shows messages sent by client and error rate.
74 102
75{{''' 103{{'''
76 title: RPC CALL sent messages 104 title: RPC CALL sent messages
77 axes: 105 axes:
78 x: time 106 x: time
79 y: sent, msg 107 y: sent, msg
108 y2: errors, msg
80 chart: line 109 chart: line
81 collection: series 110 collection: series
82 pipeline: 111 pipelines:
83 - $match: { task: omsimulator, mode: call, name: client_0 } 112 -
84 - $project: 113 - $match: { task: omsimulator, mode: call, name: client_0 }
85 x: "$seq" 114 - $project:
86 y: "$count" 115 x: "$seq"
116 y: "$count"
117 -
118 - $match: { task: omsimulator, mode: call, name: error_0 }
119 - $project:
120 x: "$seq"
121 y2: "$count"
87''' | chart_and_table 122''' | chart_and_table
88}} 123}}
89 124
90**Messages received by the server** 125Messages received by the server
126^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
127
128This chart shows messages received by the server and their latency.
91 129
92{{''' 130{{'''
93 title: RPC CALL received messages 131 title: RPC CALL received messages
@@ -106,7 +144,10 @@ RPC CALL fail-over throughput test
106''' | chart_and_table 144''' | chart_and_table
107}} 145}}
108 146
109**Round-trip messages received by the client** 147Replies received by the client
148^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
149
150This chart shows replies received by the client and total round-trip latency.
110 151
111{{''' 152{{'''
112 title: RPC CALL round-trip messages 153 title: RPC CALL round-trip messages
diff --git a/performa/scenarios/mq/omsimulator-ha-cast.rst b/performa/scenarios/mq/omsimulator-ha-cast.rst
index d996d18..26d903f 100644
--- a/performa/scenarios/mq/omsimulator-ha-cast.rst
+++ b/performa/scenarios/mq/omsimulator-ha-cast.rst
@@ -1,36 +1,51 @@
1Oslo.messaging simulator HA report 1RPC CAST fail-over test report
2---------------------------------- 2------------------------------
3 3
4This report is result of `message_queue_performance`_ execution 4This scenario is executed with help of oslo.messaging simulator. There is
5with `Oslo.messaging Simulator`_ 5one client-server pair of simulator running in single-threaded mode. The
6stats are collected from both client and server and detailed report is shown
7with one second precision.
6 8
7 9
8RPC CAST fail-over throughput test 10Execution Summary
9^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 11^^^^^^^^^^^^^^^^^
10
11**Summary**
12 12
13{{''' 13{{'''
14 title: Execution summary 14 title: RPC CAST Execution summary
15 fields: 15 fields:
16 a1: Client sent, msg 16 a1: Client sent, msg
17 b1: Server received, msg 17 a2: Server received, msg
18 b2: Loss, msg 18 a3: Loss, msg
19 c1: Avg. latency, ms 19 a4: Errors, msg
20 c2: Max latency, ms 20 b1: Duration, sec
21 c1: Throughput, msg/sec
22 d1: Transfer, Mb
23 d2: Bandwidth, Mb/sec
24 e1: Avg. latency, ms
25 e2: Max latency, ms
21 collection: records 26 collection: records
22 pipeline: 27 pipeline:
23 - $match: { task: omsimulator, mode: cast } 28 - $match: { task: omsimulator, mode: cast }
24 - $project: 29 - $project:
25 a1: "$client.count" 30 a1: "$client.count"
26 b1: "$server.count" 31 a2: "$server.count"
27 b2: { $subtract: ["$client.count", "$server.count" ] } 32 a3: { $subtract: ["$client.count", "$server.count" ] }
28 c1: { $multiply: ["$server.latency", 1000] } 33 a4: "$error.count"
29 c2: { $multiply: ["$server.max_latency", 1000] } 34 b1: "$client.duration"
35 c1: { $divide: ["$client.count", "$client.duration"] }
36 d1: { $divide: ["$client.size", 1048576] }
37 d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
38 e1: { $multiply: ["$server.latency", 1000] }
39 e2: { $multiply: ["$server.max_latency", 1000] }
30''' | info 40''' | info
31}} 41}}
32 42
33**Message flow** 43Message flow
44^^^^^^^^^^^^
45
46This chart shows the message flow between client and server. It includes
47messages sent by the client, received by the server and errors caught by
48the client.
34 49
35{{''' 50{{'''
36 title: RPC CAST message flow 51 title: RPC CAST message flow
@@ -38,7 +53,7 @@ RPC CAST fail-over throughput test
38 x: time 53 x: time
39 y1: sent, msg 54 y1: sent, msg
40 y2: received, msg 55 y2: received, msg
41 y3: latency, ms 56 y3: errors, msg
42 chart: line 57 chart: line
43 collection: series 58 collection: series
44 pipelines: 59 pipelines:
@@ -52,29 +67,51 @@ RPC CAST fail-over throughput test
52 - $project: 67 - $project:
53 x: "$timestamp" 68 x: "$timestamp"
54 y2: "$count" 69 y2: "$count"
55 y3: { $multiply: ["$latency", 1000] } 70 -
71 - $match: { task: omsimulator, mode: cast, name: error_0 }
72 - $project:
73 x: "$timestamp"
74 y3: "$count"
56''' | chart 75''' | chart
57}} 76}}
58 77
78where:
79 * ``sent`` - messages sent by the client
80 * ``received`` - messages received by the server
81 * ``errors`` - errors exposed and caught by the client
59 82
60**Messages sent by the client** 83
84Messages sent by the client
85^^^^^^^^^^^^^^^^^^^^^^^^^^^
86
87This chart shows messages sent by client and error rate.
61 88
62{{''' 89{{'''
63 title: RPC CAST sent messages 90 title: RPC CAST sent messages
64 axes: 91 axes:
65 x: time 92 x: time
66 y: sent, msg 93 y: sent, msg
94 y2: errors, msg
67 chart: line 95 chart: line
68 collection: series 96 collection: series
69 pipeline: 97 pipelines:
70 - $match: { task: omsimulator, mode: cast, name: client_0 } 98 -
71 - $project: 99 - $match: { task: omsimulator, mode: cast, name: client_0 }
72 x: "$seq" 100 - $project:
73 y: "$count" 101 x: "$seq"
102 y: "$count"
103 -
104 - $match: { task: omsimulator, mode: cast, name: error_0 }
105 - $project:
106 x: "$seq"
107 y2: "$count"
74''' | chart_and_table 108''' | chart_and_table
75}} 109}}
76 110
77**Messages received by the server** 111Messages received by the server
112^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
113
114This chart shows messages received by the server and their latency.
78 115
79{{''' 116{{'''
80 title: RPC CAST received messages 117 title: RPC CAST received messages
diff --git a/performa/scenarios/mq/omsimulator-ha-notify.rst b/performa/scenarios/mq/omsimulator-ha-notify.rst
index 83bac69..bd82f4b 100644
--- a/performa/scenarios/mq/omsimulator-ha-notify.rst
+++ b/performa/scenarios/mq/omsimulator-ha-notify.rst
@@ -1,36 +1,51 @@
1Oslo.messaging simulator HA report 1NOTIFY fail-over test report
2---------------------------------- 2----------------------------
3 3
4This report is result of `message_queue_performance`_ execution 4This scenario is executed with help of oslo.messaging simulator. There is
5with `Oslo.messaging Simulator`_ 5one client-server pair of simulator running in single-threaded mode. The
6stats are collected from both client and server and detailed report is shown
7with one second precision.
6 8
7 9
8NOTIFY fail-over throughput test 10Execution Summary
9^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 11^^^^^^^^^^^^^^^^^
10
11**Summary**
12 12
13{{''' 13{{'''
14 title: Execution summary 14 title: NOTIFY Execution summary
15 fields: 15 fields:
16 a1: Client sent, msg 16 a1: Client sent, msg
17 b1: Server received, msg 17 a2: Server received, msg
18 b2: Loss, msg 18 a3: Loss, msg
19 c1: Avg. latency, ms 19 a4: Errors, msg
20 c2: Max latency, ms 20 b1: Duration, sec
21 c1: Throughput, msg/sec
22 d1: Transfer, Mb
23 d2: Bandwidth, Mb/sec
24 e1: Avg. latency, ms
25 e2: Max latency, ms
21 collection: records 26 collection: records
22 pipeline: 27 pipeline:
23 - $match: { task: omsimulator, mode: notify } 28 - $match: { task: omsimulator, mode: notify }
24 - $project: 29 - $project:
25 a1: "$client.count" 30 a1: "$client.count"
26 b1: "$server.count" 31 a2: "$server.count"
27 b2: { $subtract: ["$client.count", "$server.count" ] } 32 a3: { $subtract: ["$client.count", "$server.count" ] }
28 c1: { $multiply: ["$server.latency", 1000] } 33 a4: "$error.count"
29 c2: { $multiply: ["$server.max_latency", 1000] } 34 b1: "$client.duration"
35 c1: { $divide: ["$client.count", "$client.duration"] }
36 d1: { $divide: ["$client.size", 1048576] }
37 d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
38 e1: { $multiply: ["$server.latency", 1000] }
39 e2: { $multiply: ["$server.max_latency", 1000] }
30''' | info 40''' | info
31}} 41}}
32 42
33**Message flow** 43Message flow
44^^^^^^^^^^^^
45
46This chart shows the message flow between client and server. It includes
47messages sent by the client, received by the server and errors caught by
48the client.
34 49
35{{''' 50{{'''
36 title: NOTIFY message flow 51 title: NOTIFY message flow
@@ -38,7 +53,7 @@ NOTIFY fail-over throughput test
38 x: time 53 x: time
39 y1: sent, msg 54 y1: sent, msg
40 y2: received, msg 55 y2: received, msg
41 y3: latency, ms 56 y3: errors, msg
42 chart: line 57 chart: line
43 collection: series 58 collection: series
44 pipelines: 59 pipelines:
@@ -52,29 +67,51 @@ NOTIFY fail-over throughput test
52 - $project: 67 - $project:
53 x: "$timestamp" 68 x: "$timestamp"
54 y2: "$count" 69 y2: "$count"
55 y3: { $multiply: ["$latency", 1000] } 70 -
71 - $match: { task: omsimulator, mode: notify, name: error_0 }
72 - $project:
73 x: "$timestamp"
74 y3: "$count"
56''' | chart 75''' | chart
57}} 76}}
58 77
78where:
79 * ``sent`` - messages sent by the client
80 * ``received`` - messages received by the server
81 * ``errors`` - errors exposed and caught by the client
59 82
60**Messages sent by the client** 83
84Messages sent by the client
85^^^^^^^^^^^^^^^^^^^^^^^^^^^
86
87This chart shows messages sent by client and error rate.
61 88
62{{''' 89{{'''
63 title: NOTIFY sent messages 90 title: NOTIFY sent messages
64 axes: 91 axes:
65 x: time 92 x: time
66 y: sent, msg 93 y: sent, msg
94 y2: errors, msg
67 chart: line 95 chart: line
68 collection: series 96 collection: series
69 pipeline: 97 pipelines:
70 - $match: { task: omsimulator, mode: notify, name: client_0 } 98 -
71 - $project: 99 - $match: { task: omsimulator, mode: notify, name: client_0 }
72 x: "$seq" 100 - $project:
73 y: "$count" 101 x: "$seq"
102 y: "$count"
103 -
104 - $match: { task: omsimulator, mode: notify, name: error_0 }
105 - $project:
106 x: "$seq"
107 y2: "$count"
74''' | chart_and_table 108''' | chart_and_table
75}} 109}}
76 110
77**Messages received by the server** 111Messages received by the server
112^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
113
114This chart shows messages received by the server and their latency.
78 115
79{{''' 116{{'''
80 title: NOTIFY received messages 117 title: NOTIFY received messages