diff --git a/performa/engine/report.py b/performa/engine/report.py index 52fe699..b0ee970 100644 --- a/performa/engine/report.py +++ b/performa/engine/report.py @@ -158,8 +158,10 @@ def generate_info(definition_str, db, doc_folder, tag): for field_name, field_title in sorted(fields.items(), key=lambda a: a[0]): value = data[field_name] + if value is None: + value = '.' patt = (''' *\n - %%s\n - %%%s\n''' % - ('d' if isinstance(value, int) else '.1f')) + ('.1f' if isinstance(value, float) else 's')) table += patt % (field_title, value) return table diff --git a/performa/modules/omsimulator.py b/performa/modules/omsimulator.py index 49647e0..07a12e7 100644 --- a/performa/modules/omsimulator.py +++ b/performa/modules/omsimulator.py @@ -121,6 +121,8 @@ def run(module): '--json %(client_file)s ' '-l %(duration)s ' '%(client_tool)s ' + '--timeout %(timeout)s ' + '-w %(sending_delay)s ' '-p %(threads)s ') % params if params['mode'] == 'cast': @@ -153,6 +155,10 @@ def run(module): round_trip_summary = client_data['summary']['round_trip'] record['round_trip'] = round_trip_summary + if 'error' in client_data['summary']: + error_summary = client_data['summary']['error'] + record['error'] = error_summary + server_summary = server_data['summary'] record['server'] = server_summary @@ -174,6 +180,8 @@ def main(): url=dict(required=True), threads=dict(type='int', default=10), duration=dict(type='int', default=10), + timeout=dict(type='int', default=5), + sending_delay=dict(type='float', default=-1.0), )) chdir(module) diff --git a/performa/scenarios/mq/omsimulator-ha-call.rst b/performa/scenarios/mq/omsimulator-ha-call.rst index df0ff43..c4027c8 100644 --- a/performa/scenarios/mq/omsimulator-ha-call.rst +++ b/performa/scenarios/mq/omsimulator-ha-call.rst @@ -1,43 +1,58 @@ -Oslo.messaging simulator HA report ----------------------------------- +RPC CALL fail-over test report +------------------------------ -This report is result of `message_queue_performance`_ execution -with `Oslo.messaging Simulator`_ +This scenario is executed with help of oslo.messaging simulator. There is +one client-server pair of simulator running in single-threaded mode. The +stats are collected from both client and server and detailed report is shown +with one second precision. -RPC CALL fail-over throughput test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -**Summary** +Execution Summary +^^^^^^^^^^^^^^^^^ {{''' - title: Execution summary + title: RPC CALL Execution summary fields: a1: Client sent, msg - b1: Server received, msg - b2: Client received replies, msg - b3: Loss, msg - c1: Avg. request latency, ms - c2: Max request latency, ms - c3: Avg. round-trip latency, ms - c4: Max round-trip latency, ms + a2: Server received, msg + a3: Client received replies, msg + a4: Loss, msg + a5: Errors, msg + b1: Duration, sec + c1: Throughput, msg/sec + d1: Transfer, Mb + d2: Bandwidth, Mb/sec + e1: Avg. request latency, ms + e2: Max request latency, ms + e3: Avg. round-trip latency, ms + e4: Max round-trip latency, ms collection: records pipeline: - $match: { task: omsimulator, mode: call } - $project: a1: "$client.count" - b1: "$server.count" - b2: "$round_trip.count" - b3: { $subtract: ["$client.count", "$round_trip.count" ] } - c1: { $multiply: ["$server.latency", 1000] } - c2: { $multiply: ["$server.max_latency", 1000] } - c3: { $multiply: ["$round_trip.latency", 1000] } - c4: { $multiply: ["$round_trip.max_latency", 1000] } + a2: "$server.count" + a3: "$round_trip.count" + a4: { $subtract: ["$client.count", "$round_trip.count" ] } + a5: "$error.count" + b1: "$client.duration" + c1: { $divide: ["$client.count", "$client.duration"] } + d1: { $divide: ["$client.size", 1048576] } + d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] } + e1: { $multiply: ["$server.latency", 1000] } + e2: { $multiply: ["$server.max_latency", 1000] } + e3: { $multiply: ["$round_trip.latency", 1000] } + e4: { $multiply: ["$round_trip.max_latency", 1000] } ''' | info }} -**Message flow** +Message flow +^^^^^^^^^^^^ + +This chart shows the message flow between client and server. It includes +messages sent by the client, received by the server, replies received by +the client and errors caught by the client. {{''' title: RPC CALL message flow @@ -46,7 +61,7 @@ RPC CALL fail-over throughput test y1: sent, msg y2: received, msg y3: round-trip, msg - y4: latency, ms + y4: errors, msg chart: line collection: series pipelines: @@ -65,29 +80,52 @@ RPC CALL fail-over throughput test - $project: x: "$timestamp" y3: "$count" - y4: { $multiply: ["$latency", 1000] } + - + - $match: { task: omsimulator, mode: call, name: error_0 } + - $project: + x: "$timestamp" + y4: "$count" ''' | chart }} +where: + * ``sent`` - messages sent by the client + * ``received`` - messages received by the server + * ``round-trip`` - replies to messages received by the client + * ``errors`` - errors exposed and caught by the client -**Messages sent by the client** + +Messages sent by the client +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows messages sent by client and error rate. {{''' title: RPC CALL sent messages axes: x: time y: sent, msg + y2: errors, msg chart: line collection: series - pipeline: - - $match: { task: omsimulator, mode: call, name: client_0 } - - $project: - x: "$seq" - y: "$count" + pipelines: + - + - $match: { task: omsimulator, mode: call, name: client_0 } + - $project: + x: "$seq" + y: "$count" + - + - $match: { task: omsimulator, mode: call, name: error_0 } + - $project: + x: "$seq" + y2: "$count" ''' | chart_and_table }} -**Messages received by the server** +Messages received by the server +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows messages received by the server and their latency. {{''' title: RPC CALL received messages @@ -106,7 +144,10 @@ RPC CALL fail-over throughput test ''' | chart_and_table }} -**Round-trip messages received by the client** +Replies received by the client +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows replies received by the client and total round-trip latency. {{''' title: RPC CALL round-trip messages diff --git a/performa/scenarios/mq/omsimulator-ha-cast.rst b/performa/scenarios/mq/omsimulator-ha-cast.rst index d996d18..26d903f 100644 --- a/performa/scenarios/mq/omsimulator-ha-cast.rst +++ b/performa/scenarios/mq/omsimulator-ha-cast.rst @@ -1,36 +1,51 @@ -Oslo.messaging simulator HA report ----------------------------------- +RPC CAST fail-over test report +------------------------------ -This report is result of `message_queue_performance`_ execution -with `Oslo.messaging Simulator`_ +This scenario is executed with help of oslo.messaging simulator. There is +one client-server pair of simulator running in single-threaded mode. The +stats are collected from both client and server and detailed report is shown +with one second precision. -RPC CAST fail-over throughput test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -**Summary** +Execution Summary +^^^^^^^^^^^^^^^^^ {{''' - title: Execution summary + title: RPC CAST Execution summary fields: a1: Client sent, msg - b1: Server received, msg - b2: Loss, msg - c1: Avg. latency, ms - c2: Max latency, ms + a2: Server received, msg + a3: Loss, msg + a4: Errors, msg + b1: Duration, sec + c1: Throughput, msg/sec + d1: Transfer, Mb + d2: Bandwidth, Mb/sec + e1: Avg. latency, ms + e2: Max latency, ms collection: records pipeline: - $match: { task: omsimulator, mode: cast } - $project: a1: "$client.count" - b1: "$server.count" - b2: { $subtract: ["$client.count", "$server.count" ] } - c1: { $multiply: ["$server.latency", 1000] } - c2: { $multiply: ["$server.max_latency", 1000] } + a2: "$server.count" + a3: { $subtract: ["$client.count", "$server.count" ] } + a4: "$error.count" + b1: "$client.duration" + c1: { $divide: ["$client.count", "$client.duration"] } + d1: { $divide: ["$client.size", 1048576] } + d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] } + e1: { $multiply: ["$server.latency", 1000] } + e2: { $multiply: ["$server.max_latency", 1000] } ''' | info }} -**Message flow** +Message flow +^^^^^^^^^^^^ + +This chart shows the message flow between client and server. It includes +messages sent by the client, received by the server and errors caught by +the client. {{''' title: RPC CAST message flow @@ -38,7 +53,7 @@ RPC CAST fail-over throughput test x: time y1: sent, msg y2: received, msg - y3: latency, ms + y3: errors, msg chart: line collection: series pipelines: @@ -52,29 +67,51 @@ RPC CAST fail-over throughput test - $project: x: "$timestamp" y2: "$count" - y3: { $multiply: ["$latency", 1000] } + - + - $match: { task: omsimulator, mode: cast, name: error_0 } + - $project: + x: "$timestamp" + y3: "$count" ''' | chart }} +where: + * ``sent`` - messages sent by the client + * ``received`` - messages received by the server + * ``errors`` - errors exposed and caught by the client -**Messages sent by the client** + +Messages sent by the client +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows messages sent by client and error rate. {{''' title: RPC CAST sent messages axes: x: time y: sent, msg + y2: errors, msg chart: line collection: series - pipeline: - - $match: { task: omsimulator, mode: cast, name: client_0 } - - $project: - x: "$seq" - y: "$count" + pipelines: + - + - $match: { task: omsimulator, mode: cast, name: client_0 } + - $project: + x: "$seq" + y: "$count" + - + - $match: { task: omsimulator, mode: cast, name: error_0 } + - $project: + x: "$seq" + y2: "$count" ''' | chart_and_table }} -**Messages received by the server** +Messages received by the server +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows messages received by the server and their latency. {{''' title: RPC CAST received messages diff --git a/performa/scenarios/mq/omsimulator-ha-notify.rst b/performa/scenarios/mq/omsimulator-ha-notify.rst index 83bac69..bd82f4b 100644 --- a/performa/scenarios/mq/omsimulator-ha-notify.rst +++ b/performa/scenarios/mq/omsimulator-ha-notify.rst @@ -1,36 +1,51 @@ -Oslo.messaging simulator HA report ----------------------------------- +NOTIFY fail-over test report +---------------------------- -This report is result of `message_queue_performance`_ execution -with `Oslo.messaging Simulator`_ +This scenario is executed with help of oslo.messaging simulator. There is +one client-server pair of simulator running in single-threaded mode. The +stats are collected from both client and server and detailed report is shown +with one second precision. -NOTIFY fail-over throughput test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -**Summary** +Execution Summary +^^^^^^^^^^^^^^^^^ {{''' - title: Execution summary + title: NOTIFY Execution summary fields: a1: Client sent, msg - b1: Server received, msg - b2: Loss, msg - c1: Avg. latency, ms - c2: Max latency, ms + a2: Server received, msg + a3: Loss, msg + a4: Errors, msg + b1: Duration, sec + c1: Throughput, msg/sec + d1: Transfer, Mb + d2: Bandwidth, Mb/sec + e1: Avg. latency, ms + e2: Max latency, ms collection: records pipeline: - $match: { task: omsimulator, mode: notify } - $project: a1: "$client.count" - b1: "$server.count" - b2: { $subtract: ["$client.count", "$server.count" ] } - c1: { $multiply: ["$server.latency", 1000] } - c2: { $multiply: ["$server.max_latency", 1000] } + a2: "$server.count" + a3: { $subtract: ["$client.count", "$server.count" ] } + a4: "$error.count" + b1: "$client.duration" + c1: { $divide: ["$client.count", "$client.duration"] } + d1: { $divide: ["$client.size", 1048576] } + d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] } + e1: { $multiply: ["$server.latency", 1000] } + e2: { $multiply: ["$server.max_latency", 1000] } ''' | info }} -**Message flow** +Message flow +^^^^^^^^^^^^ + +This chart shows the message flow between client and server. It includes +messages sent by the client, received by the server and errors caught by +the client. {{''' title: NOTIFY message flow @@ -38,7 +53,7 @@ NOTIFY fail-over throughput test x: time y1: sent, msg y2: received, msg - y3: latency, ms + y3: errors, msg chart: line collection: series pipelines: @@ -52,29 +67,51 @@ NOTIFY fail-over throughput test - $project: x: "$timestamp" y2: "$count" - y3: { $multiply: ["$latency", 1000] } + - + - $match: { task: omsimulator, mode: notify, name: error_0 } + - $project: + x: "$timestamp" + y3: "$count" ''' | chart }} +where: + * ``sent`` - messages sent by the client + * ``received`` - messages received by the server + * ``errors`` - errors exposed and caught by the client -**Messages sent by the client** + +Messages sent by the client +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows messages sent by client and error rate. {{''' title: NOTIFY sent messages axes: x: time y: sent, msg + y2: errors, msg chart: line collection: series - pipeline: - - $match: { task: omsimulator, mode: notify, name: client_0 } - - $project: - x: "$seq" - y: "$count" + pipelines: + - + - $match: { task: omsimulator, mode: notify, name: client_0 } + - $project: + x: "$seq" + y: "$count" + - + - $match: { task: omsimulator, mode: notify, name: error_0 } + - $project: + x: "$seq" + y2: "$count" ''' | chart_and_table }} -**Messages received by the server** +Messages received by the server +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This chart shows messages received by the server and their latency. {{''' title: NOTIFY received messages