Remove memory stats and optimize match operator
This commit is contained in:
parent
526a1ea35d
commit
fbc1679790
|
@ -52,11 +52,9 @@ def aggregate(scenario, mongo_url, db_name, tag):
|
|||
stop = rec['end'] # todo rename field into 'stop'
|
||||
|
||||
series_pipeline = [
|
||||
{'$match': {'$and': [
|
||||
{'tag': tag},
|
||||
{'timestamp': {'$gte': start}},
|
||||
{'timestamp': {'$lte': stop}}
|
||||
]}}
|
||||
{'$match': {
|
||||
'tag': tag,
|
||||
'timestamp': {'$gte': start, '$lte': stop}}}
|
||||
]
|
||||
series_pipeline.extend(values_pipeline)
|
||||
|
||||
|
|
|
@ -84,7 +84,7 @@ execution:
|
|||
tasks:
|
||||
- atop:
|
||||
command: stop
|
||||
labels: [ PRC, PRM ]
|
||||
labels: [ PRC ]
|
||||
|
||||
aggregation:
|
||||
-
|
||||
|
@ -95,14 +95,6 @@ aggregation:
|
|||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
|
||||
{% if rabbit_hosts[1] %}
|
||||
-
|
||||
|
@ -113,14 +105,6 @@ aggregation:
|
|||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
{% if rabbit_hosts[2] %}
|
||||
|
@ -132,14 +116,6 @@ aggregation:
|
|||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
report:
|
||||
|
|
|
@ -55,28 +55,22 @@ depending on number of concurrent threads.
|
|||
title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count
|
||||
axes:
|
||||
x: threads
|
||||
y: sent, msg/sec
|
||||
y2: received, msg/sec
|
||||
y3: round-trip, msg/sec
|
||||
y3: throughput, msg/sec
|
||||
y4: latency, ms
|
||||
y5: RabbitMQ CPU consumption, %
|
||||
y5: RabbitMQ CPU, %
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: call }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
||||
msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
|
||||
throughput: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
|
||||
latency: { $avg: "$round_trip.latency" }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y: "$msg_sent_per_sec"
|
||||
y2: "$msg_received_per_sec"
|
||||
y3: "$msg_round_trip_per_sec"
|
||||
y3: "$throughput"
|
||||
y4: { $multiply: [ "$latency", 1000 ] }
|
||||
y5: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||
''' | chart_and_table
|
||||
|
@ -120,33 +114,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
|||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ resident memory
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes memory consumption during RPC CALL load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master, Mb
|
||||
y1: Slave 1, Mb
|
||||
y2: Slave 2, Mb
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: call }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
||||
Test Case 2: RPC CAST Throughput Test
|
||||
|
@ -195,8 +162,7 @@ depending on number of concurrent threads.
|
|||
title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count
|
||||
axes:
|
||||
x: threads
|
||||
y: sent, msg/sec
|
||||
y2: received, msg/sec
|
||||
y: throughput, msg/sec
|
||||
y3: latency, ms
|
||||
y4: RabbitMQ CPU consumption, %
|
||||
chart: line
|
||||
|
@ -204,16 +170,14 @@ depending on number of concurrent threads.
|
|||
- $match: { task: omsimulator, mode: cast }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
||||
throughput: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
latency: { $avg: "$server.latency" }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y: "$msg_sent_per_sec"
|
||||
y2: "$msg_received_per_sec"
|
||||
y: "$throughput"
|
||||
y3: { $multiply: [ "$latency", 1000 ] }
|
||||
y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||
''' | chart_and_table
|
||||
|
@ -256,34 +220,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
|||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ resident memory
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes memory consumption during RPC CAST load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master, Mb
|
||||
y1: Slave 1, Mb
|
||||
y2: Slave 2, Mb
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: cast }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
||||
Test Case 3: Notification Throughput Test
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
@ -331,8 +267,7 @@ depending on number of concurrent threads.
|
|||
title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count
|
||||
axes:
|
||||
x: threads
|
||||
y: sent, msg/sec
|
||||
y2: received, msg/sec
|
||||
y: throughput, msg/sec
|
||||
y3: latency, ms
|
||||
y4: RabbitMQ CPU consumption, %
|
||||
chart: line
|
||||
|
@ -340,16 +275,14 @@ depending on number of concurrent threads.
|
|||
- $match: { task: omsimulator, mode: notify }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
||||
throughput: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||
latency: { $avg: "$server.latency" }
|
||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y: "$msg_sent_per_sec"
|
||||
y2: "$msg_received_per_sec"
|
||||
y: "$throughput"
|
||||
y3: { $multiply: [ "$latency", 1000 ] }
|
||||
y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||
''' | chart_and_table
|
||||
|
@ -392,34 +325,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
|||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
Detailed RabbitMQ resident memory
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
||||
|
||||
{{'''
|
||||
title: RabbitMQ nodes memory consumption during NOTIFY load test
|
||||
axes:
|
||||
x: threads
|
||||
y0: Master, Mb
|
||||
y1: Slave 1, Mb
|
||||
y2: Slave 2, Mb
|
||||
chart: line
|
||||
pipeline:
|
||||
- $match: { task: omsimulator, mode: notify }
|
||||
- $group:
|
||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
||||
- $project:
|
||||
x: "$_id.threads"
|
||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
||||
''' | chart_and_table
|
||||
}}
|
||||
|
||||
|
||||
.. references:
|
||||
|
||||
|
|
|
@ -87,7 +87,7 @@ execution:
|
|||
tasks:
|
||||
- atop:
|
||||
command: stop
|
||||
labels: [ PRC, PRM ]
|
||||
labels: [ PRC ]
|
||||
|
||||
aggregation:
|
||||
-
|
||||
|
@ -98,14 +98,6 @@ aggregation:
|
|||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
||||
|
||||
{% if rabbit_hosts[1] %}
|
||||
-
|
||||
|
@ -116,14 +108,6 @@ aggregation:
|
|||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||
- $group: { _id: null, rabbit_resident_1: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
{% if rabbit_hosts[2] %}
|
||||
|
@ -135,14 +119,6 @@ aggregation:
|
|||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||
-
|
||||
update:
|
||||
query:
|
||||
{ task: omsimulator }
|
||||
values:
|
||||
pipeline:
|
||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||
- $group: { _id: null, rabbit_resident_2: { $avg: "$resident" } }
|
||||
{% endif %}
|
||||
|
||||
report:
|
||||
|
|
Loading…
Reference in New Issue